level=info ts=2026-04-22T13:45:05.789771446Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T13:45:05.789794609Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-22T13:45:05.790122303Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-22T13:45:05.790893157Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T13:45:05.791270191Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T13:45:05.791338692Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.92 inf=eth0
level=info ts=2026-04-22T13:45:05.791847856Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-de7442c1
ts=2026-04-22T13:45:05.791920639Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T13:45:05.791933405Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.92 advertisePort=7946
level=debug ts=2026-04-22T13:45:05.791951768Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.92 advertisePort=7946
level=info ts=2026-04-22T13:45:05.79205388Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-22T13:45:05.796006607Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T13:45:05.796028621Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-04-22T13:45:05.797169426Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T13:45:05.797198096Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:45:05.797223019Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.169357ms
level=error ts=2026-04-22T13:45:05.79723923Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T13:45:05.797249719Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-04-22T13:45:05.797286466Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-22T13:45:05.797324582Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T13:45:05.79782607Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-22T13:45:05.797847286Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-22T13:45:05.797845266Z
level=debug ts=2026-04-22T13:45:05.797866711Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-22T13:45:05.79789575Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-22T13:45:05.797914086Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-22T13:45:05.797921372Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-22T13:45:05.797919859Z
level=debug ts=2026-04-22T13:45:05.797928213Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-22T13:45:05.799389925Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T13:45:05.79946687Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.92 inf=eth0
ts=2026-04-22T13:45:05.799905002Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:45:05.799917902Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T13:45:05.81242022Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T13:45:05.812434455Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-22T13:45:05.812446248Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-22T13:45:05.812446388Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-04-22T13:45:05.812469928Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-22T13:45:05.812465324Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-22T13:45:05.812483355Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-04-22T13:45:05.812484362Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T13:45:05.812498654Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-22T13:45:05.81250426Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-22T13:45:05.812491521Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-22T13:45:05.812994479Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-04-22T13:45:05.812996685Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-22T13:45:05.813004425Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-04-22T13:45:05.813005091Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-22T13:45:05.813016722Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-04-22T13:45:05.813054502Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-22T13:45:05.813086535Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-22T13:45:05.813090606Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-22T13:45:05.813094129Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-22T13:45:05.81309734Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-22T13:45:05.813200248Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-22T13:45:05.813205131Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-22T13:45:05.813208661Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-04-22T13:45:05.813211564Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-22T13:45:05.813222468Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-22T13:45:05.813258211Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-22T13:45:05.813328975Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=102.633µs errors=false
level=info ts=2026-04-22T13:45:05.813340643Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-22T13:45:05.813509528Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=288.682µs errors=false
level=info ts=2026-04-22T13:45:05.813518588Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-22T13:45:05.813526993Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=306.132µs
level=info ts=2026-04-22T13:45:05.813550718Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-22T13:45:05.813661228Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T13:45:05.813684975Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=debug ts=2026-04-22T13:45:05.813813215Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-22T13:45:05.813819055Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-04-22T13:45:05.813854584Z caller=loki.go:599 msg="Loki started" startup_time=83.120609ms
level=debug ts=2026-04-22T13:45:05.813906493Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T13:45:05.813955401Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=446.691862ms
level=info ts=2026-04-22T13:45:05.814069756Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
ts=2026-04-22T13:45:07.7361671Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:45:07.736207669Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:45:10.814190236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:45:11.38590344Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:45:11.385928125Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:45:15.814295752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:45:19.316291624Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:45:19.316317566Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:45:20.814770923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:45:22.198362327Z caller=logging.go:137 msg="GET /ready (503) 104.292µs"
level=warn ts=2026-04-22T13:45:25.814899313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:45:30.815075787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:45:32.011948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.93:7946"
ts=2026-04-22T13:45:32.013962836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.76:7946"
ts=2026-04-22T13:45:32.017249655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.77:7946"
level=info ts=2026-04-22T13:45:32.018536502Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.221281989s
level=debug ts=2026-04-22T13:45:32.196256907Z caller=logging.go:137 msg="GET /ready (503) 66.216µs"
level=debug ts=2026-04-22T13:45:34.727143681Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T13:45:35.814401979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-22T13:45:35.814429863Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-22T13:45:35.81443895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:45:35.814441299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:45:40.814358276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:45:45.814159241Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:45:50.814773299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:45:53.362203714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=warn ts=2026-04-22T13:45:55.815084269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:46:00.209882206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T13:46:00.557796914Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T13:46:00.557881134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:46:00.563541371Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:46:00.815088754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:46:05.797322545Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:46:05.797827185Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T13:46:05.814149522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:46:05.81418257Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:46:05.814193152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:46:05.814196592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:46:06.108094985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59366"
level=info ts=2026-04-22T13:46:10.796180132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:46:10.805914514Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:46:10.815046239Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:46:13.408711068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:46:15.814252727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:46:20.814811462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:46:23.364796774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T13:46:25.81500625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:46:27.321737556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:46:30.814454178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:46:35.614245997Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T13:46:35.813800615Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:46:35.813847109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:46:35.813858791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:46:35.814919702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:46:40.815069672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:46:45.814989426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:46:46.770823209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T13:46:47.817128757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:46:47.824023802Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:46:50.814168079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:46:53.366842051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T13:46:55.814765764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:47:00.814835434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:47:03.137803188Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T13:47:05.797417979Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:47:05.797416618Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:47:05.813778705Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:47:05.813808104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:47:05.813816665Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:47:05.814831564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:47:10.814150338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:47:15.814267338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:47:16.019493402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T13:47:16.02151262Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:47:18.347456664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:32946"
level=warn ts=2026-04-22T13:47:20.814621263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:47:23.205083943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:36282"
ts=2026-04-22T13:47:23.368071203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T13:47:25.814791718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:47:28.062623589Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:47:30.814247606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:47:35.814317872Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:47:35.814345621Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T13:47:35.81433633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:47:35.814356085Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:47:40.814157616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:47:45.814678237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:47:45.988499182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:47:45.996499053Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T13:47:47.821244441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:47:50.814631817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:47:53.370044293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:47:54.323340601Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:47:55.814625437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:48:00.814271656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:48:05.798248362Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:48:05.799257093Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:48:05.814500986Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:48:05.814534777Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:48:05.814545579Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:48:05.814594767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:48:06.116915993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41826"
level=warn ts=2026-04-22T13:48:10.814805991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:48:15.814129473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:48:20.814650449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:48:23.208794692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:59498"
ts=2026-04-22T13:48:23.372640137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:48:23.806240972Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:48:25.8142764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:48:30.814342345Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:48:32.54864151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T13:48:35.813901008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:48:35.813932869Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:48:35.813940939Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:48:35.814991361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:48:40.814121182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:48:45.490687352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:48:45.500238194Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:48:45.814163634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:48:47.982882617Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T13:48:48.354803666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:46038"
level=warn ts=2026-04-22T13:48:50.814386077Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:48:53.37493182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T13:48:55.814834488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:49:00.814481272Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:49:05.797442518Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:49:05.798346311Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:49:05.814595498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:49:05.814620052Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T13:49:05.814614222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:49:05.814630307Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:49:10.814603989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:49:15.81412978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:49:20.814314977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:49:22.809290338Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T13:49:23.214440888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:42282"
ts=2026-04-22T13:49:23.377525838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=warn ts=2026-04-22T13:49:25.814553125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:49:30.815062358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:49:33.993528187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T13:49:35.814661701Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:49:35.814689578Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:49:35.814698247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:49:35.814701965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:49:37.664986955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:49:37.671846455Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:49:40.814268866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:49:45.814279947Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:49:50.064551805Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:49:50.814686025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:49:53.217116861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:57930"
ts=2026-04-22T13:49:53.380717163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=warn ts=2026-04-22T13:49:55.814828735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:50:00.81437833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:50:05.797591689Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:50:05.798540846Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:50:05.813720815Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T13:50:05.813787279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:50:05.813809802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:50:05.813818394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T13:50:05.813868563Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-04-22T13:50:05.814140046Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-04-22T13:50:05.814174237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:50:06.126132723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33998"
level=debug ts=2026-04-22T13:50:10.169717022Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-22T13:50:10.814613458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:50:14.525619402Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-22T13:50:14.525646045Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-04-22T13:50:15.814512297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:50:18.361452009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:46202"
level=debug ts=2026-04-22T13:50:18.87939555Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=warn ts=2026-04-22T13:50:20.814158067Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:50:23.220325759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:52874"
level=debug ts=2026-04-22T13:50:23.23370601Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-22T13:50:23.38313813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:50:25.402805037Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:50:25.814105566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:50:27.589207989Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-22T13:50:30.814233505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:50:31.94409737Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=info ts=2026-04-22T13:50:34.57988848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:50:34.589655852Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T13:50:35.813885596Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:50:35.813914313Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:50:35.813923517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:50:35.814980192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:50:36.129068351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45574"
level=debug ts=2026-04-22T13:50:36.29871084Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-22T13:50:40.658641065Z caller=checkpoint.go:378 msg="writing series" size="6.2 MB"
level=debug ts=2026-04-22T13:50:40.658664831Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.5 MB" series=6
level=warn ts=2026-04-22T13:50:40.814794557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:50:42.191636638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T13:50:45.00818944Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T13:50:45.814898638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:50:49.363181237Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-22T13:50:50.81474805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:50:53.222904857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:59654"
ts=2026-04-22T13:50:53.385705681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T13:50:53.717890306Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T13:50:55.814364134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:50:58.07263579Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-04-22T13:50:58.335462182Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:51:00.815105765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:02.427275188Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-22T13:51:05.797663446Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:51:05.798874456Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:51:05.814043524Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:51:05.814069111Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:51:05.814078274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:51:05.814169039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:06.78202623Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-22T13:51:10.814798125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:11.136958538Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-22T13:51:15.491887583Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-22T13:51:15.814471717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:19.847253832Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-22T13:51:20.815109446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:51:21.541522012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:51:21.550655929Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T13:51:23.226112986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:36380"
ts=2026-04-22T13:51:23.388894316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:51:24.201910229Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=warn ts=2026-04-22T13:51:25.814586217Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:28.557314911Z caller=checkpoint.go:378 msg="writing series" size="268 kB"
level=warn ts=2026-04-22T13:51:30.814621745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:32.911269992Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=debug ts=2026-04-22T13:51:33.765225797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T13:51:35.813812007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:51:35.813849541Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:51:35.813857972Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:51:35.814860218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:37.266058014Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-22T13:51:40.814623553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:41.527392542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T13:51:41.529557743Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T13:51:41.621370396Z caller=checkpoint.go:378 msg="writing series" size="400 kB"
level=warn ts=2026-04-22T13:51:45.814209989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:45.976016345Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-04-22T13:51:45.976048411Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=15
level=debug ts=2026-04-22T13:51:50.330445728Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-22T13:51:50.814755952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:51:53.391392327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:51:54.685151115Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-22T13:51:55.814548218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:51:59.040852093Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-22T13:52:00.814684479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:03.325876768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:52:03.395308095Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-04-22T13:52:05.798286523Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:52:05.799662164Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:52:05.813799978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:52:05.813833912Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:52:05.813842613Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:52:05.814904849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:07.750362955Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-22T13:52:10.814517951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:12.105248581Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=info ts=2026-04-22T13:52:14.771172573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:52:14.778755494Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:52:15.814605828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:16.460308584Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-04-22T13:52:18.372622333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:49788"
level=debug ts=2026-04-22T13:52:20.814071928Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-22T13:52:20.814113069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:52:23.229781967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:50500"
ts=2026-04-22T13:52:23.39383352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:52:25.170156115Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=warn ts=2026-04-22T13:52:25.815084653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:29.524539892Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=debug ts=2026-04-22T13:52:29.929840749Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:52:30.220363139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T13:52:30.814835819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:33.879464959Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=info ts=2026-04-22T13:52:35.814205646Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:52:35.814261654Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:52:35.814269994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:52:35.814288466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:52:36.138778387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:36894"
level=debug ts=2026-04-22T13:52:38.234678252Z caller=checkpoint.go:378 msg="writing series" size="420 kB"
level=debug ts=2026-04-22T13:52:38.234704705Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-04-22T13:52:40.814949086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:42.58905325Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-22T13:52:45.814434453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:46.94366226Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=warn ts=2026-04-22T13:52:50.814767386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:52:51.29820415Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-04-22T13:52:53.232187816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:46286"
ts=2026-04-22T13:52:53.396196618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T13:52:55.653034928Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=warn ts=2026-04-22T13:52:55.814731405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:52:59.512700934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:52:59.525286047Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:52:59.525312672Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T13:52:59.525345115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:52:59.532508376Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T13:53:00.008229976Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=warn ts=2026-04-22T13:53:00.815085864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:04.363509119Z caller=checkpoint.go:378 msg="writing series" size="327 kB"
level=debug ts=2026-04-22T13:53:05.067687284Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T13:53:05.067758992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:53:05.075160626Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T13:53:05.79744167Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:53:05.798454715Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:53:05.814755969Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:53:05.814784366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:53:05.814792802Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:53:05.814794427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:08.717767983Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-04-22T13:53:10.814657389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:11.769259169Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T13:53:13.072208761Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-04-22T13:53:13.570554157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:53:13.577258089Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:53:15.814180725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:17.427701793Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-22T13:53:20.81420523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:21.78221317Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
ts=2026-04-22T13:53:23.397535902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=info ts=2026-04-22T13:53:23.703846581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:53:23.711611368Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T13:53:25.814682524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:26.137305895Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=debug ts=2026-04-22T13:53:30.492384196Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=debug ts=2026-04-22T13:53:30.492413492Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-22T13:53:30.814546091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:34.847004901Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-04-22T13:53:35.814031734Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:53:35.814074474Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:53:35.81408526Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:53:35.814140163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:53:36.143201261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37200"
level=debug ts=2026-04-22T13:53:39.202017447Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-22T13:53:39.835969013Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:53:40.814390541Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:43.557135472Z caller=checkpoint.go:378 msg="writing series" size="421 kB"
level=info ts=2026-04-22T13:53:45.02491236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:53:45.036337867Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:53:45.814278417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:53:46.564228831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:53:46.571726576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T13:53:47.911351502Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=warn ts=2026-04-22T13:53:50.814918155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:52.266114755Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
ts=2026-04-22T13:53:53.399140462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T13:53:55.814855445Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:53:56.620861397Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=warn ts=2026-04-22T13:54:00.815066969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:54:00.977210508Z caller=checkpoint.go:378 msg="writing series" size="3.5 MB"
level=debug ts=2026-04-22T13:54:00.977235816Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.3 MB" series=7
level=debug ts=2026-04-22T13:54:02.447319972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T13:54:05.330542015Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-22T13:54:05.797359737Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:54:05.797924168Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:54:05.81427413Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T13:54:05.814280982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:54:05.814300068Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:54:05.814308365Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-22T13:54:06.145718915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:36396"
level=debug ts=2026-04-22T13:54:09.68604941Z caller=checkpoint.go:378 msg="writing series" size="365 kB"
level=warn ts=2026-04-22T13:54:10.814708069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:54:11.641890563Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:54:14.040636135Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=warn ts=2026-04-22T13:54:15.814148961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:54:18.395608605Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-22T13:54:20.814363873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:54:22.75045748Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
ts=2026-04-22T13:54:23.401784769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T13:54:25.815091575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:54:27.105275405Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-22T13:54:30.814385418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:54:31.460156483Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T13:54:35.813820479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:54:35.813861709Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:54:35.813873382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:54:35.814931094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:54:35.814938357Z caller=checkpoint.go:390 msg="flushing series" totalSize="798 kB" series=7
level=info ts=2026-04-22T13:54:35.819338956Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-22T13:54:35.825489011Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.011479953s
ts=2026-04-22T13:54:36.148813216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49562"
level=debug ts=2026-04-22T13:54:38.138125741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:54:40.814633507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:54:40.884962432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:54:40.897079222Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T13:54:42.512157004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:54:42.519722351Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T13:54:45.814660942Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:54:48.385511901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:60576"
level=warn ts=2026-04-22T13:54:50.814627595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:54:53.404227484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=warn ts=2026-04-22T13:54:55.814765161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:55:00.814695214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:02.211831612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T13:55:03.924537272Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T13:55:05.797371043Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:55:05.79744056Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:55:05.813742613Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T13:55:05.813809392Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:55:05.813829515Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:55:05.813838574Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T13:55:05.813877138Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=warn ts=2026-04-22T13:55:05.814156636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:05.814243458Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
level=debug ts=2026-04-22T13:55:09.844803694Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-22T13:55:10.814626925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:13.875018161Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-22T13:55:13.875045749Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-04-22T13:55:15.814111917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:17.904683357Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
ts=2026-04-22T13:55:18.386869757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:41292"
level=warn ts=2026-04-22T13:55:20.81441556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:21.93387038Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-22T13:55:23.406084197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=info ts=2026-04-22T13:55:23.946143436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:55:23.956656227Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:55:25.814139984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:25.964031807Z caller=checkpoint.go:378 msg="writing series" size="259 kB"
level=debug ts=2026-04-22T13:55:29.994195269Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=warn ts=2026-04-22T13:55:30.814939518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:30.931401815Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:55:34.023329051Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T13:55:35.81393428Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:55:35.813966573Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:55:35.813976616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:55:35.815066827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:55:36.153253419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40034"
level=info ts=2026-04-22T13:55:37.150514075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:55:37.164436069Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T13:55:38.05778365Z caller=checkpoint.go:378 msg="writing series" size="7.0 MB"
level=debug ts=2026-04-22T13:55:38.057808904Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.5 MB" series=6
level=warn ts=2026-04-22T13:55:40.814984456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:42.083018944Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-22T13:55:42.856646999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T13:55:42.85872017Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:55:45.814125715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:46.113086412Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-22T13:55:50.143255078Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T13:55:50.814489034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:55:53.408255729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T13:55:54.172813575Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T13:55:55.814804413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:55:58.150272371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:55:58.202733917Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-22T13:56:00.814910989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:02.233055684Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB"
level=info ts=2026-04-22T13:56:05.797846733Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:56:05.799874986Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:56:05.814076018Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:56:05.814108122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:56:05.814119531Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:56:05.814152826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:56:06.156371648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54772"
level=debug ts=2026-04-22T13:56:06.262476261Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-04-22T13:56:10.292608405Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-22T13:56:10.814523072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:14.321879186Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-22T13:56:15.814426356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:18.351902237Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-04-22T13:56:19.632598056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:56:19.646209473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:56:20.814447763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:22.381781073Z caller=checkpoint.go:378 msg="writing series" size="255 kB"
ts=2026-04-22T13:56:23.411319872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T13:56:25.815104984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:26.16081207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T13:56:26.41133615Z caller=checkpoint.go:378 msg="writing series" size="211 kB"
level=debug ts=2026-04-22T13:56:28.322385783Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T13:56:30.14047001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:56:30.148981664Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T13:56:30.44190909Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-04-22T13:56:30.814354749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:34.471501505Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=info ts=2026-04-22T13:56:35.814373861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:56:35.814407719Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T13:56:35.814401654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:56:35.814416591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T13:56:38.501212734Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-22T13:56:40.814139015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:42.531499834Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-04-22T13:56:42.531522174Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=warn ts=2026-04-22T13:56:45.814644142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:46.561063557Z caller=checkpoint.go:378 msg="writing series" size="83 kB"
level=debug ts=2026-04-22T13:56:50.591180254Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-22T13:56:50.814143692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:56:53.41354683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T13:56:54.620625173Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-04-22T13:56:55.815124011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:56:58.650805138Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-22T13:57:00.814683761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:57:01.149699903Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:57:02.68060788Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=info ts=2026-04-22T13:57:05.797970782Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:57:05.799350861Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:57:05.814577277Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:57:05.81461263Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:57:05.814620546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:57:05.814630688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:57:06.161374749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57562"
level=debug ts=2026-04-22T13:57:06.710201479Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
level=debug ts=2026-04-22T13:57:08.896732662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T13:57:10.740328764Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-22T13:57:10.814513196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:57:14.02378293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:57:14.032873681Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T13:57:14.770468347Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-22T13:57:15.814135442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:57:16.369452985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:57:16.376765897Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T13:57:18.800053294Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=warn ts=2026-04-22T13:57:20.814888176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:57:22.830051833Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-22T13:57:23.253358638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:50428"
ts=2026-04-22T13:57:23.41540561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T13:57:25.814361042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:57:26.859488222Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-22T13:57:29.599993769Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:57:30.814841397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:57:30.889873593Z caller=checkpoint.go:378 msg="writing series" size="420 kB"
level=debug ts=2026-04-22T13:57:30.889904612Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=12
level=debug ts=2026-04-22T13:57:34.918753457Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=info ts=2026-04-22T13:57:35.813991227Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:57:35.814023737Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:57:35.814033057Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:57:35.815103464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:57:36.164220966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59460"
level=debug ts=2026-04-22T13:57:38.949248754Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=warn ts=2026-04-22T13:57:40.814801784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:57:42.979071709Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=warn ts=2026-04-22T13:57:45.814846718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:57:47.009088721Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-04-22T13:57:48.399668067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44990"
level=debug ts=2026-04-22T13:57:48.947828758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T13:57:50.814647181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:57:51.038151036Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
ts=2026-04-22T13:57:53.255965738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:36554"
ts=2026-04-22T13:57:53.418066819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T13:57:55.067990892Z caller=checkpoint.go:378 msg="writing series" size="8.1 kB"
level=warn ts=2026-04-22T13:57:55.814703885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:57:55.866026988Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:57:59.098602287Z caller=checkpoint.go:378 msg="writing series" size="241 kB"
level=info ts=2026-04-22T13:57:59.515833718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:57:59.525897498Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T13:58:00.814903022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:03.128515277Z caller=checkpoint.go:378 msg="writing series" size="315 kB"
level=debug ts=2026-04-22T13:58:03.128541793Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-04-22T13:58:05.797665722Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:58:05.79815094Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:58:05.814514722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:58:05.814535685Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:58:05.814543476Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:58:05.814531666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:07.157864742Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=info ts=2026-04-22T13:58:08.475710891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:58:08.483046853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T13:58:10.814776121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:11.188202209Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=debug ts=2026-04-22T13:58:15.218191664Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-22T13:58:15.814111498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:19.247290501Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-22T13:58:20.814126572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:23.277608576Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-22T13:58:23.421254178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T13:58:23.994280043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T13:58:25.814997715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:27.307047974Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=warn ts=2026-04-22T13:58:30.814815009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:31.33704226Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=debug ts=2026-04-22T13:58:35.367116446Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=info ts=2026-04-22T13:58:35.814749978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T13:58:35.814760749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:35.814781545Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:58:35.814790625Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T13:58:37.582455221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T13:58:39.396643848Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-22T13:58:40.814807486Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:43.427153427Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T13:58:45.814446486Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:47.457074674Z caller=checkpoint.go:378 msg="writing series" size="635 kB"
level=debug ts=2026-04-22T13:58:47.457098886Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=11
ts=2026-04-22T13:58:48.40384378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:39118"
level=warn ts=2026-04-22T13:58:50.814201165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:51.07718575Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:58:51.486682788Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
ts=2026-04-22T13:58:53.259722227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41300"
level=info ts=2026-04-22T13:58:53.320457896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:58:53.330509322Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T13:58:53.423201335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T13:58:55.516734055Z caller=checkpoint.go:378 msg="writing series" size="308 kB"
level=warn ts=2026-04-22T13:58:55.814907499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:58:59.546511654Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=warn ts=2026-04-22T13:59:00.814156375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:59:03.581852432Z caller=checkpoint.go:378 msg="writing series" size="7.7 MB"
level=debug ts=2026-04-22T13:59:03.581883882Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.0 MB" series=4
level=info ts=2026-04-22T13:59:05.79810151Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T13:59:05.798851396Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T13:59:05.814062915Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T13:59:05.814095715Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:59:05.814104619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T13:59:05.814147312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:59:06.358814948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:59:06.370235397Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T13:59:07.605812708Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=warn ts=2026-04-22T13:59:10.814505006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:59:11.635936652Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=debug ts=2026-04-22T13:59:15.665978145Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-22T13:59:15.814677649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:59:18.404960818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:49816"
level=debug ts=2026-04-22T13:59:19.695060934Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-22T13:59:20.814118601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:59:23.263012702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:42884"
ts=2026-04-22T13:59:23.425324951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T13:59:23.725318757Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=debug ts=2026-04-22T13:59:25.038747618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T13:59:25.040618167Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:59:25.814177813Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:59:26.423748469Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T13:59:27.755574289Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-22T13:59:30.814267595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:59:31.784381526Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T13:59:34.936409061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:59:34.945185979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:59:34.945209554Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T13:59:34.945235698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:59:34.951514242Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T13:59:35.814607675Z caller=checkpoint.go:390 msg="flushing series" totalSize="504 kB" series=7
level=info ts=2026-04-22T13:59:35.814641701Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T13:59:35.814640875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T13:59:35.814661929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T13:59:35.814670975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T13:59:35.817329133Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-22T13:59:35.820871491Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006736231s
level=warn ts=2026-04-22T13:59:40.814445437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T13:59:45.814495234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T13:59:47.259088442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T13:59:47.269536339Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T13:59:48.405977124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:38834"
level=info ts=2026-04-22T13:59:49.149459913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:59:49.157025935Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:59:49.157047678Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T13:59:49.157071312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T13:59:49.163918764Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T13:59:50.814239779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T13:59:53.266349548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:44320"
ts=2026-04-22T13:59:53.428547963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=warn ts=2026-04-22T13:59:55.814874349Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:00.712841774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:00:00.814530125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:00:05.797910542Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:00:05.798526273Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-04-22T14:00:05.798727196Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T14:00:05.798771259Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974295 n=1
level=debug ts=2026-04-22T14:00:05.79880972Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974295 n=1
level=info ts=2026-04-22T14:00:05.813738528Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T14:00:05.81380257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:00:05.813828706Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:00:05.813837479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T14:00:05.81387291Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-22T14:00:05.814096989Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-22T14:00:05.814113539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:00:06.173648825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54730"
level=info ts=2026-04-22T14:00:08.515032571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:00:08.523299344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:00:09.844761395Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-04-22T14:00:10.573771581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:00:10.580980076Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:00:10.815074822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:13.874978909Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-22T14:00:13.875008922Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-04-22T14:00:15.814956604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:17.90479508Z caller=checkpoint.go:378 msg="writing series" size="496 kB"
level=debug ts=2026-04-22T14:00:18.254662111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T14:00:20.814339248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:21.933571778Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-22T14:00:23.431415287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:00:24.728610408Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:00:25.814430124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:25.963999132Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=info ts=2026-04-22T14:00:29.207158711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:00:29.216035595Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:00:29.993682123Z caller=checkpoint.go:378 msg="writing series" size="270 kB"
level=warn ts=2026-04-22T14:00:30.814350093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:34.023463473Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T14:00:35.813938553Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:00:35.813978427Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:00:35.813988087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:00:35.815099561Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:38.056803532Z caller=checkpoint.go:378 msg="writing series" size="7.7 MB"
level=debug ts=2026-04-22T14:00:38.056825123Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.7 MB" series=6
level=warn ts=2026-04-22T14:00:40.814764788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:42.083139615Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T14:00:45.814161132Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:46.113587698Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-22T14:00:50.142524389Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T14:00:50.814219246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:00:53.433845744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:00:53.727793737Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:00:54.159774421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:00:54.168254318Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:00:54.172469617Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T14:00:55.814633951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:00:55.970803828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:00:58.202938989Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-22T14:01:00.814305905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:02.232168967Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=info ts=2026-04-22T14:01:05.797864291Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:01:05.798539834Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:01:05.81383104Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:01:05.813863267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:01:05.813883787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:01:05.814832354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:06.262317186Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-04-22T14:01:10.292599175Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-22T14:01:10.766432558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:01:10.774334641Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:01:10.814671074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:14.321793055Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-22T14:01:15.814183548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:18.352207305Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-22T14:01:18.412683803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:59558"
level=warn ts=2026-04-22T14:01:20.815065947Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:21.167824019Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:01:22.382071796Z caller=checkpoint.go:378 msg="writing series" size="255 kB"
ts=2026-04-22T14:01:23.437437364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T14:01:25.814783968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:26.411692978Z caller=checkpoint.go:378 msg="writing series" size="263 kB"
level=debug ts=2026-04-22T14:01:30.44133015Z caller=checkpoint.go:378 msg="writing series" size="155 kB"
level=warn ts=2026-04-22T14:01:30.814977934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:34.471214726Z caller=checkpoint.go:378 msg="writing series" size="394 kB"
level=debug ts=2026-04-22T14:01:34.471241269Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=14
level=info ts=2026-04-22T14:01:35.814768781Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:01:35.814799211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:01:35.81481102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:01:35.814790786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:38.179835352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T14:01:38.296277767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:01:38.306126247Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:01:38.501909822Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-22T14:01:38.501938942Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=1
level=warn ts=2026-04-22T14:01:40.814525858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:42.531267595Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-22T14:01:45.815056487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:01:46.56129461Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
ts=2026-04-22T14:01:48.414052423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:58760"
level=debug ts=2026-04-22T14:01:50.539972735Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:01:50.590786244Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-22T14:01:50.814504652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:01:53.440002861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:01:54.62012546Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-04-22T14:01:55.814720937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:01:56.753753608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:01:56.763309795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:01:58.650636719Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-04-22T14:02:00.814134597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:02:02.680079347Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=info ts=2026-04-22T14:02:05.797540353Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:02:05.797587869Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:02:05.813833625Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:02:05.813865203Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:02:05.813884163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:02:05.814959207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:02:06.185689351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39514"
level=debug ts=2026-04-22T14:02:06.709837992Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=debug ts=2026-04-22T14:02:10.740525956Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
level=warn ts=2026-04-22T14:02:10.81428209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:02:14.276752642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:02:14.770156956Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-22T14:02:15.815018536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:02:18.800030732Z caller=checkpoint.go:378 msg="writing series" size="395 kB"
level=debug ts=2026-04-22T14:02:18.800069181Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-22T14:02:20.814835438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:02:22.829234512Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-22T14:02:23.44223909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T14:02:23.57826625Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:02:25.814660728Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:02:25.917048647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.5 MB" total_uncomp="44 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:02:25.927319456Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:02:26.859664587Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-22T14:02:30.814308125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:02:30.889689941Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=debug ts=2026-04-22T14:02:34.919534871Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=info ts=2026-04-22T14:02:35.813931038Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:02:35.813984642Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:02:35.813997686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:02:35.81506823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:02:36.188777372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33788"
level=debug ts=2026-04-22T14:02:38.949450646Z caller=checkpoint.go:378 msg="writing series" size="215 kB"
level=warn ts=2026-04-22T14:02:40.814958437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:02:42.978784767Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=debug ts=2026-04-22T14:02:42.97880794Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-04-22T14:02:45.814308955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:02:46.51240798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:02:46.525923969Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:02:47.008629194Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-04-22T14:02:48.418072692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:36452"
level=warn ts=2026-04-22T14:02:50.814251827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:02:51.038803924Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=debug ts=2026-04-22T14:02:53.114889414Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T14:02:53.444941647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:02:54.359886755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:02:54.36180439Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T14:02:55.067910234Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-22T14:02:55.814138643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:02:59.097959463Z caller=checkpoint.go:378 msg="writing series" size="395 kB"
level=warn ts=2026-04-22T14:03:00.81487999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:03.128382402Z caller=checkpoint.go:378 msg="writing series" size="456 kB"
level=debug ts=2026-04-22T14:03:03.128409962Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-22T14:03:05.797952856Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:03:05.799149203Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:03:05.814439522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:03:05.814466337Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:03:05.814475685Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:03:05.814468458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:07.158138866Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=warn ts=2026-04-22T14:03:10.814155218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:03:11.110788657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:03:11.118338984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:03:11.187674754Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=debug ts=2026-04-22T14:03:15.217842764Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-22T14:03:15.815041359Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:19.247129317Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-22T14:03:20.815068446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:22.646165535Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:03:23.277377071Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
ts=2026-04-22T14:03:23.284126012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:46960"
ts=2026-04-22T14:03:23.448290026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=warn ts=2026-04-22T14:03:25.814508755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:27.306653522Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=warn ts=2026-04-22T14:03:30.814139212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:31.336717577Z caller=checkpoint.go:378 msg="writing series" size="277 kB"
level=info ts=2026-04-22T14:03:31.625456396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:03:31.635650521Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:03:35.36684991Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=info ts=2026-04-22T14:03:35.813784755Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:03:35.813819932Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:03:35.813828892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:03:35.814896511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:38.757654776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:03:39.39671756Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=warn ts=2026-04-22T14:03:40.814394715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:43.426939316Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T14:03:45.814734393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:47.4571779Z caller=checkpoint.go:378 msg="writing series" size="643 kB"
level=debug ts=2026-04-22T14:03:47.457203159Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=11
ts=2026-04-22T14:03:48.422031818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:36148"
level=debug ts=2026-04-22T14:03:49.801725324Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:03:50.81417188Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:51.485955654Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
ts=2026-04-22T14:03:53.45048168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T14:03:55.516233717Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-22T14:03:55.815074192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:03:59.546121776Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=warn ts=2026-04-22T14:04:00.814969744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:04:03.582980281Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-22T14:04:03.583011036Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=4
level=info ts=2026-04-22T14:04:04.323167324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:04:04.333010962Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T14:04:05.797365217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:04:05.797881626Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:04:05.814137102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:04:05.814165376Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:04:05.814176208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:04:05.814176572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:04:07.605593505Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=warn ts=2026-04-22T14:04:10.815057727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:04:11.636145458Z caller=checkpoint.go:378 msg="writing series" size="292 kB"
level=debug ts=2026-04-22T14:04:14.278462616Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:04:15.665332023Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=warn ts=2026-04-22T14:04:15.814317369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:04:19.695448465Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-22T14:04:20.814679865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:04:23.289553104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:33010"
ts=2026-04-22T14:04:23.452295315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:04:23.72535253Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=debug ts=2026-04-22T14:04:24.391767864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T14:04:25.815000604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:04:27.755095626Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=info ts=2026-04-22T14:04:29.387587106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:04:29.395175756Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:04:30.814633335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:04:31.784525193Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T14:04:35.81446204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T14:04:35.814478531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:04:35.8144911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:04:35.814499339Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T14:04:35.814488491Z caller=checkpoint.go:390 msg="flushing series" totalSize="646 kB" series=7
level=info ts=2026-04-22T14:04:35.820096864Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-22T14:04:35.823540621Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009506712s
level=warn ts=2026-04-22T14:04:40.814253955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T14:04:45.81452012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:04:48.056231201Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T14:04:48.426776902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:45342"
level=warn ts=2026-04-22T14:04:50.814213357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:04:53.292574566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:60476"
ts=2026-04-22T14:04:53.455426809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=warn ts=2026-04-22T14:04:55.814874397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:05:00.29198266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:05:00.300053284Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:05:00.814208226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:03.754171475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T14:05:05.797386299Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:05:05.797425702Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:05:05.813757944Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T14:05:05.813791584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:05:05.813809326Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:05:05.813834994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T14:05:05.81392977Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=warn ts=2026-04-22T14:05:05.814146935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:05.814158446Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=debug ts=2026-04-22T14:05:09.844396254Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-04-22T14:05:10.815049631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:13.875124657Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-22T14:05:13.87514889Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=3
level=warn ts=2026-04-22T14:05:15.814418106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:17.904595439Z caller=checkpoint.go:378 msg="writing series" size="596 kB"
level=warn ts=2026-04-22T14:05:20.814167006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:21.934113005Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T14:05:22.640268736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T14:05:23.458283737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=info ts=2026-04-22T14:05:23.621143111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:05:23.629496145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:05:25.814600914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:25.963858586Z caller=checkpoint.go:378 msg="writing series" size="408 kB"
level=debug ts=2026-04-22T14:05:29.99371868Z caller=checkpoint.go:378 msg="writing series" size="338 kB"
level=debug ts=2026-04-22T14:05:29.993743682Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-04-22T14:05:30.814219997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:34.02308885Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T14:05:35.814487936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:05:35.814517078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:05:35.814525348Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:05:35.814531014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:05:36.203863238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40234"
level=debug ts=2026-04-22T14:05:38.057584748Z caller=checkpoint.go:378 msg="writing series" size="8.4 MB"
level=debug ts=2026-04-22T14:05:38.057608889Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.4 MB" series=2
level=warn ts=2026-04-22T14:05:40.814621864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:42.083321196Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T14:05:45.814399812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:46.113368825Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-22T14:05:47.699732101Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T14:05:48.431946751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:34794"
level=debug ts=2026-04-22T14:05:50.093679997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:05:50.142851967Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T14:05:50.814681253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:05:51.29785859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:05:51.305851269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:05:51.305875372Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:05:51.305901808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:05:51.312297787Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T14:05:53.460499378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T14:05:54.172428694Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T14:05:55.815075657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:05:58.202235228Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-22T14:06:00.814457317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:02.232990013Z caller=checkpoint.go:378 msg="writing series" size="7.6 kB"
level=info ts=2026-04-22T14:06:04.5923041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:06:04.604577907Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:06:04.604606154Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:06:04.604641988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:06:04.611642105Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T14:06:05.621226006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:06:05.627781466Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:06:05.79728065Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:06:05.798341271Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T14:06:05.814782552Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:06:05.814810739Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:06:05.814819435Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:06:05.814862823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:06:06.205988462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39186"
level=debug ts=2026-04-22T14:06:06.262684949Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=debug ts=2026-04-22T14:06:10.292766688Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-22T14:06:10.814949602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:14.32198963Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-22T14:06:14.43944951Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:06:15.814412016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:18.351494873Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
ts=2026-04-22T14:06:18.433415639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:45126"
level=info ts=2026-04-22T14:06:19.833194484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:06:19.845807292Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:06:20.814990487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:22.381743814Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
ts=2026-04-22T14:06:23.461549723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T14:06:25.814113042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:26.411319869Z caller=checkpoint.go:378 msg="writing series" size="424 kB"
level=debug ts=2026-04-22T14:06:30.178286023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:06:30.181716396Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T14:06:30.441811951Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=warn ts=2026-04-22T14:06:30.814713038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:34.471410685Z caller=checkpoint.go:378 msg="writing series" size="286 kB"
level=debug ts=2026-04-22T14:06:34.471431449Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=14
level=warn ts=2026-04-22T14:06:35.814251461Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:06:35.814252999Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:06:35.814280593Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:06:35.814289193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T14:06:36.501410157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:06:36.50896704Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:06:38.502313014Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-22T14:06:38.502337795Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=1
level=warn ts=2026-04-22T14:06:40.814715202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:42.530652404Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-22T14:06:45.815069196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:46.560924045Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-04-22T14:06:48.749117146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:06:48.758774332Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:06:49.012914373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:06:50.590787528Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-22T14:06:50.814520946Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:06:53.464125936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:06:54.620149631Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-22T14:06:55.814869327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:06:58.650582654Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-04-22T14:07:00.814399603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:02.680728062Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-22T14:07:05.798321594Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:07:05.800190118Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:07:05.814511176Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:07:05.814602927Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T14:07:05.814595929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:07:05.814616167Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T14:07:06.710154107Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=debug ts=2026-04-22T14:07:10.73984492Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-22T14:07:10.814251449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:13.421467468Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:07:14.770197572Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-22T14:07:15.814519195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:18.799366024Z caller=checkpoint.go:378 msg="writing series" size="302 kB"
level=warn ts=2026-04-22T14:07:20.815086623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:22.829708214Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=debug ts=2026-04-22T14:07:22.829739513Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-04-22T14:07:23.355154762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T14:07:23.386232715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:07:23.395058506Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T14:07:23.466516424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T14:07:25.814931313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:26.859002934Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-22T14:07:30.814715186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:30.889249312Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=info ts=2026-04-22T14:07:32.395964716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:07:32.403955563Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:07:34.919020435Z caller=checkpoint.go:378 msg="writing series" size="290 kB"
level=info ts=2026-04-22T14:07:35.813808685Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:07:35.813861799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:07:35.813870744Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:07:35.814842383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:38.9495509Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=warn ts=2026-04-22T14:07:40.814631023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:42.810328505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:07:42.978495294Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=debug ts=2026-04-22T14:07:42.978526805Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-04-22T14:07:45.814847456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:47.00860554Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-22T14:07:50.814177285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:51.038138286Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
ts=2026-04-22T14:07:53.468298128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:07:55.068619432Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-22T14:07:55.814390339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:07:59.098079292Z caller=checkpoint.go:378 msg="writing series" size="296 kB"
level=warn ts=2026-04-22T14:08:00.814315263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:03.128014146Z caller=checkpoint.go:378 msg="writing series" size="508 kB"
level=info ts=2026-04-22T14:08:05.798091618Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:08:05.799040684Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:08:05.814461321Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:08:05.814492478Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:08:05.814503905Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:08:05.814538873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:07.15851069Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=debug ts=2026-04-22T14:08:07.15853359Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-04-22T14:08:07.949782114Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:08:10.23330937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T14:08:10.395003173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:08:10.403218471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:08:10.814895467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:11.18815319Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=debug ts=2026-04-22T14:08:15.217534772Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-22T14:08:15.814322289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:19.247607606Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-22T14:08:20.814627764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:23.277975143Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
ts=2026-04-22T14:08:23.470794414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T14:08:25.81504276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:08:27.112903739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:08:27.12218393Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:08:27.306864045Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-22T14:08:30.814778164Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:31.337203863Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=debug ts=2026-04-22T14:08:35.366585373Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=info ts=2026-04-22T14:08:35.814005335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:08:35.814031869Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:08:35.814042857Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:08:35.815142689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:39.396242346Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-22T14:08:40.814580671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:42.61314728Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:08:43.426592095Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T14:08:45.815100324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:47.456887095Z caller=checkpoint.go:378 msg="writing series" size="694 kB"
level=debug ts=2026-04-22T14:08:47.456910997Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=10
level=warn ts=2026-04-22T14:08:50.814779381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:51.4860498Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
ts=2026-04-22T14:08:53.472933039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=info ts=2026-04-22T14:08:55.257228943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:08:55.269204088Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:08:55.516190851Z caller=checkpoint.go:378 msg="writing series" size="314 kB"
level=warn ts=2026-04-22T14:08:55.81483508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:08:59.546484268Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=warn ts=2026-04-22T14:09:00.814169274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:09:02.975746601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:09:03.586089597Z caller=checkpoint.go:378 msg="writing series" size="13 MB"
level=debug ts=2026-04-22T14:09:03.586114037Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=4
level=debug ts=2026-04-22T14:09:05.797320282Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:09:05.79837558Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T14:09:05.813785908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:09:05.813809334Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:09:05.813816972Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:09:05.814831371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:09:07.605505279Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=warn ts=2026-04-22T14:09:10.814611972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:09:11.636027005Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=debug ts=2026-04-22T14:09:14.806967786Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:09:15.516857176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:09:15.525522353Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:09:15.664938809Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-22T14:09:15.814609346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:09:19.6949619Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-22T14:09:20.814128091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:09:23.475331146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:09:23.725338108Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=warn ts=2026-04-22T14:09:25.814479105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:09:27.755097916Z caller=checkpoint.go:378 msg="writing series" size="149 kB"
level=warn ts=2026-04-22T14:09:30.814505859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:09:31.785121485Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T14:09:35.813924809Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:09:35.813951695Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:09:35.81396042Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:09:35.814994951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:09:35.814998284Z caller=checkpoint.go:390 msg="flushing series" totalSize="785 kB" series=7
level=info ts=2026-04-22T14:09:35.821288812Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-22T14:09:35.825698054Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.011640136s
level=warn ts=2026-04-22T14:09:40.814798322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T14:09:45.814808553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:09:47.365436702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:09:47.37617195Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:09:47.886627753Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:09:50.814303526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:09:53.317134099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:40104"
ts=2026-04-22T14:09:53.477962327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T14:09:55.814591357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T14:10:00.815069427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:01.607590768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:10:05.797191655Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:10:05.79823604Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T14:10:05.813637147Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T14:10:05.813771036Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-22T14:10:05.813784684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:10:05.813805526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:10:05.813827823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T14:10:05.814023334Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=warn ts=2026-04-22T14:10:05.814692143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:10:08.819356184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:10:08.826100215Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:10:09.84451376Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=warn ts=2026-04-22T14:10:10.814117098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:13.874937424Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-22T14:10:13.874965274Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=3
level=warn ts=2026-04-22T14:10:15.814690884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:17.90473069Z caller=checkpoint.go:378 msg="writing series" size="630 kB"
level=warn ts=2026-04-22T14:10:20.814220152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:21.93415689Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T14:10:23.062848628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T14:10:23.320485791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48230"
ts=2026-04-22T14:10:23.480393483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T14:10:25.814423827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:25.963357699Z caller=checkpoint.go:378 msg="writing series" size="334 kB"
level=debug ts=2026-04-22T14:10:29.993483881Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
level=debug ts=2026-04-22T14:10:29.993508369Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-04-22T14:10:30.814370985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:34.023513229Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T14:10:35.814834618Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T14:10:35.814834552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:35.814861478Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:10:35.814869884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T14:10:38.059164274Z caller=checkpoint.go:378 msg="writing series" size="9.0 MB"
level=debug ts=2026-04-22T14:10:38.059189923Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.0 MB" series=2
level=warn ts=2026-04-22T14:10:40.814176445Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:42.082987523Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-22T14:10:45.420004045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:10:45.427722127Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:10:45.814145315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:46.112692878Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-22T14:10:48.458840795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:42352"
level=debug ts=2026-04-22T14:10:50.143174501Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T14:10:50.814700113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:50.980102226Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:10:50.980165685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:10:50.988519061Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-psuf\"}"
level=info ts=2026-04-22T14:10:53.230193649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:10:53.237526119Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T14:10:53.482176912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:10:54.172524208Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T14:10:55.814586904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:10:56.730119266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:10:56.73165405Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T14:10:58.202944818Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-22T14:11:00.814608924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:02.232559036Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=info ts=2026-04-22T14:11:05.557904847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:11:05.567885524Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-psuf\"}"
level=info ts=2026-04-22T14:11:05.797484233Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:11:05.797503797Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:11:05.813938474Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:11:05.813974896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:11:05.813983358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:11:05.815044346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:06.262501427Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=debug ts=2026-04-22T14:11:10.292530715Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-22T14:11:10.815054828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:14.322415757Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T14:11:15.814888765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:18.351599671Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T14:11:20.814616896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:22.381829899Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
ts=2026-04-22T14:11:23.324274342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:57290"
ts=2026-04-22T14:11:23.484166449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T14:11:25.361256938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:11:25.814158205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:26.411892595Z caller=checkpoint.go:378 msg="writing series" size="324 kB"
level=info ts=2026-04-22T14:11:26.419825866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:11:26.42810915Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:11:30.441131942Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=warn ts=2026-04-22T14:11:30.814979921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:34.471245187Z caller=checkpoint.go:378 msg="writing series" size="381 kB"
level=debug ts=2026-04-22T14:11:34.471270609Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=14
level=info ts=2026-04-22T14:11:35.813807721Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:11:35.813842374Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:11:35.813852157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:11:35.814825584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:37.489171915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:11:38.502481288Z caller=checkpoint.go:378 msg="writing series" size="3.0 MB"
level=debug ts=2026-04-22T14:11:38.502505905Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=1
level=warn ts=2026-04-22T14:11:40.81454402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:42.530915527Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=info ts=2026-04-22T14:11:43.13223145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:11:43.139413525Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-psuf\"}"
level=info ts=2026-04-22T14:11:44.971810961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:11:44.981984755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:11:45.814297966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:46.560514441Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=debug ts=2026-04-22T14:11:50.591664225Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-22T14:11:50.814376825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:11:53.487710618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:11:54.356894783Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:11:54.620817626Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-22T14:11:55.814176857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:11:58.649949601Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=warn ts=2026-04-22T14:12:00.814986369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:02.680760973Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=info ts=2026-04-22T14:12:05.797614183Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:12:05.80426298Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:12:05.813824225Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:12:05.813855606Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:12:05.81386462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:12:05.814843014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:06.709854767Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=debug ts=2026-04-22T14:12:10.739841279Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-04-22T14:12:10.814930363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:14.769851948Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-22T14:12:15.814986428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:18.799821201Z caller=checkpoint.go:378 msg="writing series" size="432 kB"
level=debug ts=2026-04-22T14:12:18.799857031Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-04-22T14:12:20.814167382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:12:22.68573149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:12:22.697992051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:12:22.69802559Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:12:22.698065863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:12:22.706698638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:12:22.829986478Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-22T14:12:23.490957613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T14:12:25.814177752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:26.859410401Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=debug ts=2026-04-22T14:12:28.518935224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:12:30.814812592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:30.888839405Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=info ts=2026-04-22T14:12:32.824617955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:12:32.831381192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T14:12:32.866466431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:12:32.873679539Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:12:33.438755158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:12:34.919449342Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-04-22T14:12:35.814413054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T14:12:35.81441878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:35.814442341Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:12:35.814450899Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T14:12:38.802351428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:12:38.811169313Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:12:38.811190617Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:12:38.811218615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:12:38.818487057Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:12:38.948896059Z caller=checkpoint.go:378 msg="writing series" size="250 kB"
level=warn ts=2026-04-22T14:12:40.81415431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:42.979832499Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=debug ts=2026-04-22T14:12:42.979866376Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-04-22T14:12:45.814166647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:47.008219773Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-04-22T14:12:48.469100405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:49158"
level=warn ts=2026-04-22T14:12:50.814452914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:51.038845341Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=info ts=2026-04-22T14:12:51.187524641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:12:51.197700778Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T14:12:53.332639772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:48834"
ts=2026-04-22T14:12:53.493951093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:12:55.068210124Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-22T14:12:55.815021382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:12:59.0986704Z caller=checkpoint.go:378 msg="writing series" size="393 kB"
level=debug ts=2026-04-22T14:12:59.245019767Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:13:00.81469945Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:03.128081307Z caller=checkpoint.go:378 msg="writing series" size="697 kB"
level=debug ts=2026-04-22T14:13:03.128106491Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-22T14:13:05.797731916Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:13:05.798900256Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:13:05.814313747Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:13:05.814349825Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:13:05.814359806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:13:05.814358182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:13:06.241430677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39324"
level=debug ts=2026-04-22T14:13:07.157938764Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=info ts=2026-04-22T14:13:10.175894145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:13:10.188236736Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:13:10.814144597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:11.188045462Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=info ts=2026-04-22T14:13:14.786540608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:13:14.797036488Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:13:15.217144302Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-22T14:13:15.814519947Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:17.845938444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:13:19.247509406Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-22T14:13:20.814361405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:23.281152675Z caller=checkpoint.go:378 msg="writing series" size="385 kB"
ts=2026-04-22T14:13:23.496872487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=debug ts=2026-04-22T14:13:24.85925521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:13:25.814789984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:27.307583527Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=info ts=2026-04-22T14:13:27.958599624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:13:27.969022042Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:13:30.81449844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:31.336939924Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-22T14:13:35.366386905Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=debug ts=2026-04-22T14:13:35.366411473Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-22T14:13:35.813814691Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:13:35.813845565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:13:35.813853656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:13:35.81485168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:39.396830369Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-22T14:13:40.814488214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:43.426253764Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T14:13:45.814105662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:47.456181071Z caller=checkpoint.go:378 msg="writing series" size="939 kB"
level=debug ts=2026-04-22T14:13:49.356643904Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:13:50.814939038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:51.485656521Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
ts=2026-04-22T14:13:53.338041919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:40670"
ts=2026-04-22T14:13:53.500022551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:13:55.516468828Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=debug ts=2026-04-22T14:13:55.516500308Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-04-22T14:13:55.814403744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:13:59.54591722Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=warn ts=2026-04-22T14:14:00.814960075Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:14:03.593915433Z caller=checkpoint.go:378 msg="writing series" size="17 MB"
level=debug ts=2026-04-22T14:14:03.59394081Z caller=checkpoint.go:390 msg="flushing series" totalSize="17 MB" series=2
level=info ts=2026-04-22T14:14:05.798057883Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:14:05.798997764Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:14:05.814372175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:14:05.814405073Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:14:05.814412981Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:14:05.814450238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:14:07.375686692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:14:07.383881457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:14:07.605748585Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=warn ts=2026-04-22T14:14:10.814278172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:14:11.635635653Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=info ts=2026-04-22T14:14:13.898411508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:14:13.907416343Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:14:14.657260283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:14:15.665269823Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=warn ts=2026-04-22T14:14:15.815095579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:14:18.85396754Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:14:19.695021804Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-22T14:14:20.814410099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:14:23.502755558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:14:23.724707675Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=info ts=2026-04-22T14:14:25.3463694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:14:25.356789843Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:14:25.815076558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:14:27.755185721Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=warn ts=2026-04-22T14:14:30.814816426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:14:31.785182982Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=debug ts=2026-04-22T14:14:35.814556987Z caller=checkpoint.go:390 msg="flushing series" totalSize="958 kB" series=7
level=info ts=2026-04-22T14:14:35.81455692Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:14:35.81460624Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:14:35.814617131Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:14:35.814636353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:14:35.820531734Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-22T14:14:35.821351688Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007429423s
level=warn ts=2026-04-22T14:14:40.814766122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T14:14:45.814459956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T14:14:50.815077911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:14:53.342544517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:44736"
ts=2026-04-22T14:14:53.504462836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T14:14:54.229917417Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:14:54.22998327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:14:54.238992655Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T14:14:55.814702655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:14:56.219269834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:14:56.228505867Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-psuf\"}"
level=warn ts=2026-04-22T14:15:00.81423512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:04.374255266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:15:04.37618921Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T14:15:05.797580727Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:15:05.797614104Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=debug ts=2026-04-22T14:15:05.798923196Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T14:15:05.798974634Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974296 n=1
level=debug ts=2026-04-22T14:15:05.799018202Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974296 n=1
level=info ts=2026-04-22T14:15:05.814021502Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T14:15:05.814058046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:15:05.814082974Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:15:05.814091426Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:15:05.814123283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:15:05.814163477Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-04-22T14:15:05.81446661Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=info ts=2026-04-22T14:15:07.256596986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:15:07.282817265Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T14:15:08.411456313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:15:08.418028071Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:15:09.845169889Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-22T14:15:10.815079119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:13.875520949Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-22T14:15:13.87554527Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=3
level=info ts=2026-04-22T14:15:15.110404237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="145 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:15:15.117659743Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:15:15.814169193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:17.904692191Z caller=checkpoint.go:378 msg="writing series" size="656 kB"
level=debug ts=2026-04-22T14:15:19.623291358Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:15:20.814610083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:21.934841381Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-22T14:15:23.50676825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T14:15:25.814314646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:25.965381294Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
level=debug ts=2026-04-22T14:15:29.994668078Z caller=checkpoint.go:378 msg="writing series" size="471 kB"
level=debug ts=2026-04-22T14:15:29.994699214Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-04-22T14:15:30.814281242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:34.023815521Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T14:15:35.814610551Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:15:35.814640626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:15:35.814649233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:15:35.814636859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:15:36.497770939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:15:36.509881377Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:15:38.058303517Z caller=checkpoint.go:378 msg="writing series" size="9.3 MB"
level=debug ts=2026-04-22T14:15:38.058327964Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.3 MB" series=2
level=info ts=2026-04-22T14:15:40.105877678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:15:40.112198005Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T14:15:40.81415937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:42.083352651Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T14:15:45.814844567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:46.113666834Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-22T14:15:48.555112232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:15:48.568456377Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:15:50.143558244Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T14:15:50.814472845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:15:53.509225656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:15:54.173505026Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-04-22T14:15:54.781494323Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:15:55.814529517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:15:58.202557013Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=debug ts=2026-04-22T14:15:58.82758958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T14:16:00.814508279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:16:02.232547801Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-22T14:16:05.797371779Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:16:05.797703752Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:16:05.813810971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:16:05.813838878Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:16:05.813850533Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:16:05.81494972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:16:06.262436359Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-04-22T14:16:10.292459213Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-22T14:16:10.814145814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:16:11.939179908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="145 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:16:11.948459746Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:16:14.322024551Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T14:16:15.814304087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:16:18.351839043Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
ts=2026-04-22T14:16:18.488071501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:57046"
level=info ts=2026-04-22T14:16:20.30939478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:16:20.317451427Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T14:16:20.814696451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:16:22.381852094Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
ts=2026-04-22T14:16:23.348729432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41444"
ts=2026-04-22T14:16:23.510800069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T14:16:25.384997917Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:16:25.814503181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:16:26.412016023Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
level=debug ts=2026-04-22T14:16:30.44188884Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=warn ts=2026-04-22T14:16:30.814158011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:16:33.951625258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:16:33.963241529Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:16:34.471661462Z caller=checkpoint.go:378 msg="writing series" size="494 kB"
level=debug ts=2026-04-22T14:16:34.4716873Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=14
level=info ts=2026-04-22T14:16:35.813797112Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:16:35.8138279Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:16:35.813836281Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:16:35.814871291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:16:36.258768802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60750"
level=debug ts=2026-04-22T14:16:38.504930012Z caller=checkpoint.go:378 msg="writing series" size="6.6 MB"
level=debug ts=2026-04-22T14:16:38.504960326Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.6 MB" series=1
level=warn ts=2026-04-22T14:16:40.814499888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:16:41.474699334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:16:41.48277067Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:16:42.53146432Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-22T14:16:45.814437871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:16:46.561136566Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-22T14:16:50.590689635Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-22T14:16:50.814973229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:16:52.388889095Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:16:52.958679881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:16:53.351324451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:35300"
ts=2026-04-22T14:16:53.513500771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:16:54.621343486Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-22T14:16:55.814699687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:16:58.650504629Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=warn ts=2026-04-22T14:17:00.81448867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:17:02.105133353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="145 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:17:02.112598178Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T14:17:02.125600884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:17:02.130803219Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:17:02.680900185Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=info ts=2026-04-22T14:17:05.79738146Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:17:05.797898251Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:17:05.814454365Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:17:05.81449673Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:17:05.814508404Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:17:05.814464292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:06.71023593Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=debug ts=2026-04-22T14:17:10.740043904Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T14:17:10.814834045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:14.769910536Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-22T14:17:15.814266285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:17.784660502Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:17:18.799974331Z caller=checkpoint.go:378 msg="writing series" size="323 kB"
level=debug ts=2026-04-22T14:17:18.799999239Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-22T14:17:20.814651344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:17:22.619237049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:17:22.632835046Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:17:22.632863391Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:17:22.632899736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:17:22.640212081Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:17:22.830400498Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-22T14:17:23.515693617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=info ts=2026-04-22T14:17:24.386946493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:17:24.394880843Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:17:25.814813806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:26.859987624Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=warn ts=2026-04-22T14:17:30.81420105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:30.890182387Z caller=checkpoint.go:378 msg="writing series" size="389 kB"
level=debug ts=2026-04-22T14:17:34.91995186Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=info ts=2026-04-22T14:17:35.81416105Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T14:17:35.814174899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:35.814195939Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:17:35.814206599Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T14:17:37.299836792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:17:37.31021682Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:17:38.949317381Z caller=checkpoint.go:378 msg="writing series" size="250 kB"
level=warn ts=2026-04-22T14:17:40.814357864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:42.979799818Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=debug ts=2026-04-22T14:17:42.979823996Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-04-22T14:17:45.814594215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:47.009058837Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-22T14:17:50.814958555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:51.03888103Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=info ts=2026-04-22T14:17:52.242530561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="145 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:17:52.250769517Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:17:52.835294523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:17:52.849943203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:17:53.517312322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=info ts=2026-04-22T14:17:53.91833978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:17:53.923832771Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:17:55.068200866Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=warn ts=2026-04-22T14:17:55.814837554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:17:59.098999705Z caller=checkpoint.go:378 msg="writing series" size="292 kB"
level=warn ts=2026-04-22T14:18:00.814636208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:03.129037889Z caller=checkpoint.go:378 msg="writing series" size="886 kB"
level=debug ts=2026-04-22T14:18:03.129068805Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-22T14:18:05.797821195Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:18:05.798764438Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:18:05.814216101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:18:05.81424054Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:18:05.814248721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:18:05.814258527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:18:06.265071249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48982"
level=debug ts=2026-04-22T14:18:07.158311397Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=warn ts=2026-04-22T14:18:10.814800601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:11.188328186Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=info ts=2026-04-22T14:18:14.139920807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:18:14.147493794Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:18:15.218265906Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-22T14:18:15.814833431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:18:17.084952503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:18:17.09166558Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T14:18:18.497823123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:38156"
level=debug ts=2026-04-22T14:18:19.247870492Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=debug ts=2026-04-22T14:18:20.232819634Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:18:20.814954853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:23.278102686Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
ts=2026-04-22T14:18:23.518918278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T14:18:25.814757288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:27.307712011Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-22T14:18:30.814587093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:31.337237821Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=debug ts=2026-04-22T14:18:31.337262228Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-22T14:18:33.423991533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="145 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:18:33.43448696Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:18:35.367159801Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=info ts=2026-04-22T14:18:35.814535617Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:18:35.814600853Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T14:18:35.81459087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:18:35.814611631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T14:18:36.914198132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:18:36.921543326Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:18:39.396670583Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-22T14:18:40.814710871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:42.355042024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:18:43.426558608Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T14:18:45.814245431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:47.456708066Z caller=checkpoint.go:378 msg="writing series" size="928 kB"
level=debug ts=2026-04-22T14:18:47.45673428Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-04-22T14:18:50.124012424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:18:50.814707254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:51.487088719Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
ts=2026-04-22T14:18:53.520783153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=info ts=2026-04-22T14:18:54.720634331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:18:54.72963331Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:18:55.515988026Z caller=checkpoint.go:378 msg="writing series" size="331 kB"
level=warn ts=2026-04-22T14:18:55.814197588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:18:59.5460867Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=warn ts=2026-04-22T14:19:00.814372091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:19:03.593725257Z caller=checkpoint.go:378 msg="writing series" size="20 MB"
level=debug ts=2026-04-22T14:19:03.593748754Z caller=checkpoint.go:390 msg="flushing series" totalSize="20 MB" series=4
level=info ts=2026-04-22T14:19:05.797750231Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:19:05.798525123Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:19:05.813823247Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:19:05.813851085Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:19:05.813862685Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:19:05.814950664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:19:07.605990574Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-04-22T14:19:10.814916575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:19:11.131000321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:19:11.140395945Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:19:11.140420086Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:19:11.140447673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:19:11.146856822Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:19:11.636642196Z caller=checkpoint.go:378 msg="writing series" size="581 kB"
level=debug ts=2026-04-22T14:19:15.665650136Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
level=warn ts=2026-04-22T14:19:15.814617039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:19:19.695120693Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-22T14:19:20.8143168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:19:23.523475242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:19:23.725444963Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=info ts=2026-04-22T14:19:25.104775291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:19:25.11397708Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T14:19:25.814694452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:19:26.04299428Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:19:26.043055975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="318 kB" avg_comp="318 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:19:26.047368324Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-22T14:19:27.755792233Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=debug ts=2026-04-22T14:19:27.755821624Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-04-22T14:19:29.067579313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:19:29.075325901Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T14:19:29.910864044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="145 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:19:29.918492073Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:19:29.918517119Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:19:29.918548706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="145 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:19:29.925904006Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:19:30.814231686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:19:31.784720965Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T14:19:35.814316203Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:19:35.814342546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:19:35.814350892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:19:35.8143138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:19:35.814376633Z caller=checkpoint.go:390 msg="flushing series" totalSize="47 kB" series=1
level=info ts=2026-04-22T14:19:35.817988301Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-22T14:19:35.818993361Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.004641314s
ts=2026-04-22T14:19:36.27337086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54862"
level=info ts=2026-04-22T14:19:39.065847405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:19:39.070254541Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-22T14:19:40.210456737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:19:40.218030925Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=warn ts=2026-04-22T14:19:40.814442119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:19:41.417699267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="145 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:19:41.427402198Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:19:44.773995652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:19:44.775655783Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T14:19:45.814468071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:19:48.504343018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:57332"
level=warn ts=2026-04-22T14:19:50.814185197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:19:52.511429697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:19:52.521161892Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T14:19:53.525976522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T14:19:55.814133708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:19:57.649341822Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:19:59.256461934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:19:59.261100966Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-22T14:20:00.814276343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:20:05.797697826Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:20:05.798537706Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:20:05.813876771Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:20:05.813906227Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:20:05.813914998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T14:20:05.813920005Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T14:20:05.814087221Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=warn ts=2026-04-22T14:20:05.814133451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:20:05.814384651Z caller=checkpoint.go:378 msg="writing series" size="267 kB"
ts=2026-04-22T14:20:06.27588265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49688"
level=debug ts=2026-04-22T14:20:09.845296035Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=warn ts=2026-04-22T14:20:10.814713008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:20:10.885634117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:20:10.894121388Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:20:13.876374051Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-22T14:20:13.876404253Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=3
level=warn ts=2026-04-22T14:20:15.814433898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:20:17.905094451Z caller=checkpoint.go:378 msg="writing series" size="775 kB"
level=info ts=2026-04-22T14:20:20.155196021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:20:20.163618248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:20:20.814213371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:20:21.162044767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:20:21.169336831Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:20:21.934464058Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-22T14:20:23.527534713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T14:20:25.814160333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:20:25.923934298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:20:25.964223199Z caller=checkpoint.go:378 msg="writing series" size="248 kB"
level=debug ts=2026-04-22T14:20:29.993875102Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
level=debug ts=2026-04-22T14:20:29.993897524Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-04-22T14:20:30.814523263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:20:32.69753868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:20:34.023528133Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T14:20:35.814282332Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:20:35.814324409Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:20:35.814333116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:20:35.814351267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:20:36.278680916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48358"
level=debug ts=2026-04-22T14:20:38.063304756Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-22T14:20:38.063330016Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=2
level=warn ts=2026-04-22T14:20:40.815047396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:20:42.083261518Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-22T14:20:42.841131873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:20:42.850120121Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:20:45.814630554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:20:46.113257606Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-22T14:20:46.857224746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:20:46.862013815Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-22T14:20:48.508424822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:53428"
level=debug ts=2026-04-22T14:20:50.143016585Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T14:20:50.815074101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:20:53.53015946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:20:54.173502734Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T14:20:55.814250975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:20:57.497164607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:20:57.505347496Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:20:57.505370413Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:20:57.505397549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:20:57.510910587Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:20:58.203062517Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-22T14:21:00.815132339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:02.232736751Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-22T14:21:04.276121686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:21:05.797417835Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:21:05.797454151Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:21:05.813876091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:21:05.813902115Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:21:05.813911655Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:21:05.815007836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:06.26307747Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
ts=2026-04-22T14:21:06.282005617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37824"
level=info ts=2026-04-22T14:21:08.864156646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:21:08.873638928Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:21:10.292672801Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-22T14:21:10.356784393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:21:10.364508787Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=warn ts=2026-04-22T14:21:10.814367256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:14.32274168Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-22T14:21:14.63069024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:21:14.637909446Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T14:21:15.814993758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:18.192903994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:21:18.35222776Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-22T14:21:18.509666004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44520"
level=warn ts=2026-04-22T14:21:20.814476271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:22.382162551Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
ts=2026-04-22T14:21:23.532857409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=warn ts=2026-04-22T14:21:25.814507489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:26.412233099Z caller=checkpoint.go:378 msg="writing series" size="540 kB"
level=debug ts=2026-04-22T14:21:30.442053465Z caller=checkpoint.go:378 msg="writing series" size="136 kB"
level=debug ts=2026-04-22T14:21:30.44207802Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-04-22T14:21:30.814687808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:21:33.50778644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:21:33.517465468Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:21:34.472411221Z caller=checkpoint.go:378 msg="writing series" size="383 kB"
level=info ts=2026-04-22T14:21:35.814136354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:21:35.814172977Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:21:35.814184364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:21:35.81419325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:38.508402667Z caller=checkpoint.go:378 msg="writing series" size="7.8 MB"
level=debug ts=2026-04-22T14:21:38.508427251Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.2 MB" series=2
level=debug ts=2026-04-22T14:21:39.30369091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:21:40.278111378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:21:40.283350739Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-22T14:21:40.814666539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:42.531167448Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-22T14:21:45.814426214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:46.561690767Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=debug ts=2026-04-22T14:21:50.59110832Z caller=checkpoint.go:378 msg="writing series" size="643 B"
level=warn ts=2026-04-22T14:21:50.814948041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:21:53.375735492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:40438"
ts=2026-04-22T14:21:53.533976515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=info ts=2026-04-22T14:21:54.065296465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:21:54.074725768Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:21:54.620914523Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-22T14:21:55.814780221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:21:58.650786335Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=warn ts=2026-04-22T14:22:00.814763165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:22:02.418533769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:22:02.427640827Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:22:02.680379326Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-04-22T14:22:02.880047362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:22:02.890610785Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T14:22:05.79741089Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:22:05.798256471Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:22:05.814688107Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:22:05.814720087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:22:05.814731066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:22:05.814776747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:22:06.288637901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:34614"
level=debug ts=2026-04-22T14:22:06.710001348Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=debug ts=2026-04-22T14:22:09.790289931Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:22:10.740077779Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-22T14:22:10.814870262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:14.770506828Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-22T14:22:15.814179811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:18.414437694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:22:18.799960902Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=debug ts=2026-04-22T14:22:18.799991565Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-04-22T14:22:19.084333434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:22:19.090883521Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:22:20.814314712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:22.829986565Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-22T14:22:23.378828291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:36358"
ts=2026-04-22T14:22:23.536138643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=info ts=2026-04-22T14:22:23.873803973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:22:23.879280332Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-22T14:22:25.81488892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:26.860039794Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-22T14:22:30.814788402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:30.891671016Z caller=checkpoint.go:378 msg="writing series" size="358 kB"
level=debug ts=2026-04-22T14:22:34.919948528Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=info ts=2026-04-22T14:22:35.813957494Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:22:35.813988568Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:22:35.81399758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:22:35.815096862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:36.720916357Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:22:38.948886145Z caller=checkpoint.go:378 msg="writing series" size="250 kB"
level=debug ts=2026-04-22T14:22:38.948909035Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-22T14:22:40.814683505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:22:42.579652509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:22:42.604381288Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:22:42.97989689Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=warn ts=2026-04-22T14:22:45.815195473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:47.009326127Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-22T14:22:50.814217072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:51.038792781Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
ts=2026-04-22T14:22:53.538483015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=info ts=2026-04-22T14:22:53.697659037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:22:53.710641325Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:22:55.068455615Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-22T14:22:55.815089782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:22:56.121843271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:22:59.100863344Z caller=checkpoint.go:378 msg="writing series" size="423 kB"
level=warn ts=2026-04-22T14:23:00.81509708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:23:01.322777918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:23:01.329862305Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:23:03.129016523Z caller=checkpoint.go:378 msg="writing series" size="891 kB"
level=debug ts=2026-04-22T14:23:03.129039711Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=6
level=info ts=2026-04-22T14:23:05.798804826Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:23:05.800815573Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:23:05.814319402Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:23:05.814349531Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:23:05.814357798Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:23:05.81435627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:23:06.294331098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52228"
level=debug ts=2026-04-22T14:23:07.158539708Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=warn ts=2026-04-22T14:23:10.81475951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:23:11.189710629Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=debug ts=2026-04-22T14:23:11.811894275Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:23:15.217868512Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-22T14:23:15.81435256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:23:18.010803383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:23:18.019659973Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:23:19.247857532Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-22T14:23:20.81415132Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:23:21.642789127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:23:21.647125273Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-22T14:23:23.28292136Z caller=checkpoint.go:378 msg="writing series" size="468 kB"
ts=2026-04-22T14:23:23.540732636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T14:23:25.814848785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:23:27.307065577Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T14:23:30.814217533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:23:31.337030535Z caller=checkpoint.go:378 msg="writing series" size="317 kB"
level=debug ts=2026-04-22T14:23:31.337052307Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-04-22T14:23:35.366884406Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=info ts=2026-04-22T14:23:35.81383044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:23:35.813866753Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:23:35.813877944Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:23:35.815002092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:23:36.799732522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:23:36.810892904Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:23:36.810923274Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:23:36.810962482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:23:36.817881996Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:23:37.334858392Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T14:23:37.899641117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:23:37.906615988Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T14:23:39.396778782Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=warn ts=2026-04-22T14:23:40.814976348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:23:43.426806179Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-22T14:23:45.814901141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:23:47.457351906Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-22T14:23:47.457374806Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-04-22T14:23:48.112837445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:23:48.120900876Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:23:49.335459172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:23:49.337356134Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T14:23:50.815116864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:23:51.492687587Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
ts=2026-04-22T14:23:53.543934249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=info ts=2026-04-22T14:23:54.690658818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:23:54.699169141Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:23:55.516915Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=warn ts=2026-04-22T14:23:55.814584436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:23:59.545839205Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=warn ts=2026-04-22T14:24:00.815040475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:24:03.601233017Z caller=checkpoint.go:378 msg="writing series" size="22 MB"
level=debug ts=2026-04-22T14:24:03.601267362Z caller=checkpoint.go:390 msg="flushing series" totalSize="22 MB" series=4
level=info ts=2026-04-22T14:24:04.79673736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:24:04.806343787Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T14:24:05.800599689Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:24:05.802602391Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:24:05.813820762Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:24:05.81385289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:24:05.813868163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:24:05.814919935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:24:06.997764964Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:24:07.605807939Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=warn ts=2026-04-22T14:24:10.814803469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:24:11.635523318Z caller=checkpoint.go:378 msg="writing series" size="676 kB"
level=info ts=2026-04-22T14:24:14.314790307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:24:14.320274606Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-22T14:24:15.665987552Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
level=warn ts=2026-04-22T14:24:15.815041608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:24:18.526402179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:41796"
level=debug ts=2026-04-22T14:24:19.696698302Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-22T14:24:20.814891581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:24:23.546058627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=debug ts=2026-04-22T14:24:23.725462278Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=info ts=2026-04-22T14:24:25.341842283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:24:25.351915291Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-psuf\"}"
level=warn ts=2026-04-22T14:24:25.814201293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:24:25.922427918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T14:24:27.755005168Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-22T14:24:27.755029586Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-04-22T14:24:30.814889446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:24:31.788774475Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-22T14:24:34.689635192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:24:34.701956913Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T14:24:35.813784342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:24:35.813826929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:24:35.813836572Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T14:24:35.814865059Z caller=checkpoint.go:390 msg="flushing series" totalSize="47 kB" series=1
level=warn ts=2026-04-22T14:24:35.814887163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:24:35.820062929Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-22T14:24:35.82110548Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006866316s
ts=2026-04-22T14:24:36.304539466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51436"
level=debug ts=2026-04-22T14:24:37.095901727Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:24:40.814679951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T14:24:45.81425926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T14:24:50.814996481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:24:53.340807619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:24:53.349205027Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T14:24:53.403884237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:35408"
ts=2026-04-22T14:24:53.54796311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=warn ts=2026-04-22T14:24:55.814868009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T14:25:00.814947929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:25:03.523682811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:25:03.531137408Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T14:25:04.906515633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:25:04.911071084Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-22T14:25:05.797368063Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T14:25:05.797867906Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:25:05.814254882Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T14:25:05.814300396Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:25:05.814321982Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:25:05.814330365Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:25:05.814351383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:25:05.814383722Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-04-22T14:25:05.814596017Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=debug ts=2026-04-22T14:25:09.845522506Z caller=checkpoint.go:378 msg="writing series" size="155 kB"
level=debug ts=2026-04-22T14:25:09.898887137Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:25:10.814329418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:25:11.160762665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T14:25:11.185684738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:25:11.194044561Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:25:13.875857265Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-22T14:25:13.875881538Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=3
level=warn ts=2026-04-22T14:25:15.814395783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:25:17.90464411Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-22T14:25:17.904666993Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=1
level=warn ts=2026-04-22T14:25:20.814137978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:25:21.934510336Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-22T14:25:22.417255984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:25:22.427839029Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-22T14:25:23.551413679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-fz4x9-faba4edb 10.130.0.77:7946"
level=warn ts=2026-04-22T14:25:25.814876985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:25:25.964628485Z caller=checkpoint.go:378 msg="writing series" size="392 kB"
level=debug ts=2026-04-22T14:25:29.994400434Z caller=checkpoint.go:378 msg="writing series" size="488 kB"
level=warn ts=2026-04-22T14:25:30.814175331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:25:34.023780875Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T14:25:35.814072304Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:25:35.81410203Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T14:25:35.814109828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:25:35.814119501Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-22T14:25:36.309941459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:43920"
level=debug ts=2026-04-22T14:25:38.059957753Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-22T14:25:38.059987822Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=5
level=debug ts=2026-04-22T14:25:40.067220117Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T14:25:40.814128873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:25:41.669799913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:25:41.678113675Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:25:42.083252866Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-22T14:25:45.814723605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:25:46.113289045Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-22T14:25:48.377318745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:25:48.384123251Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:25:48.384144631Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:25:48.384169995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:25:48.390215675Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T14:25:50.143903834Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T14:25:50.814922707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:25:53.408057582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.77:41222"
level=info ts=2026-04-22T14:25:53.515835216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:25:53.521899986Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:25:53.521920484Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:25:53.521943218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:25:53.525711874Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-22T14:25:53.55388599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:25:54.172871331Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T14:25:55.814374889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:25:58.203387596Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=info ts=2026-04-22T14:25:59.144208409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:25:59.150536918Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T14:26:00.815105898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:26:01.210657354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T14:26:01.947851095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:26:01.954897985Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:26:02.232745288Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-22T14:26:05.797267937Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.67:11211 count=1
level=info ts=2026-04-22T14:26:05.798304776Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T14:26:05.814592501Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:26:05.814616676Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:26:05.814625311Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:26:05.814652474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:26:06.26242241Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-04-22T14:26:09.704125247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:26:09.708455237Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-22T14:26:10.292387397Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-22T14:26:10.814686549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:26:11.700601395Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:26:14.322980462Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T14:26:15.814350949Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:26:18.352104618Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-04-22T14:26:19.865441732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T14:26:19.873582937Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db56ff542%3A19db58b1115%3A7776e23d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T14:26:20.814280544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:26:22.382777766Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=info ts=2026-04-22T14:26:23.028477208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.7 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T14:26:23.035636198Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db56ffc82%3A19db570a058%3Ac2fb574b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T14:26:23.55627127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c07d562a 10.128.0.93:7946"
level=info ts=2026-04-22T14:26:23.89011058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:26:23.898439856Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:26:23.898465029Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T14:26:23.898496845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="169 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:26:23.905667408Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T14:26:25.814219442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:26:26.412682892Z caller=checkpoint.go:378 msg="writing series" size="442 kB"
level=debug ts=2026-04-22T14:26:30.441934999Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-22T14:26:30.814903136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T14:26:34.472094911Z caller=checkpoint.go:378 msg="writing series" size="481 kB"
level=debug ts=2026-04-22T14:26:34.472121941Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=14
level=info ts=2026-04-22T14:26:35.814679011Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T14:26:35.814708615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T14:26:35.814718393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T14:26:35.81471193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:26:36.314210524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56468"
level=debug ts=2026-04-22T14:26:37.524949569Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T14:26:38.507028696Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-22T14:26:38.507052747Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=1
level=warn ts=2026-04-22T14:26:40.814799104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:26:40.817888805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="193 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T14:26:40.826830449Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db57059b4%3A19db576d753%3A256cdb03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T14:26:42.531501094Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-22T14:26:45.814557279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T14:26:45.957699412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26b2485a2b4da816 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-psuf\"}"
level=error ts=2026-04-22T14:26:45.9666069Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26b2485a2b4da816/19db575a89d%3A19db5870aa2%3A5fe6f513?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-psuf\"}"
level=debug ts=2026-04-22T14:26:46.561764816Z caller=checkpoint.go:378 msg="writing series" size="119 kB"
level=info ts=2026-04-22T14:26:49.234473689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="326 kB" avg_comp="326 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T14:26:49.23900005Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db56ff751%3A19db58a52f6%3Aa99db2f9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-22T14:26:50.591685153Z caller=checkpoint.go:378 msg="writing series" size="624 B"
level=warn ts=2026-04-22T14:26:50.814389102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T14:26:53.558628511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7rdl9-ea25fd23 10.130.0.76:7946"
level=debug ts=2026-04-22T14:26:54.620589361Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-22T14:26:55.814759241Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"