level=info ts=2026-04-19T19:19:03.015481556Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-19T19:19:03.016094194Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-19T19:19:03.016734133Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-19T19:19:03.017559308Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-19T19:19:03.017987978Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T19:19:03.018055175Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.69 inf=eth0
level=info ts=2026-04-19T19:19:03.018609371Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-ec3bce6d
ts=2026-04-19T19:19:03.018678016Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-19T19:19:03.018693037Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.69 advertisePort=7946
level=debug ts=2026-04-19T19:19:03.018715Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.69 advertisePort=7946
level=info ts=2026-04-19T19:19:03.018852673Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-19T19:19:03.022295624Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T19:19:03.022333457Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-04-19T19:19:03.023444629Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T19:19:03.023468532Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:19:03.023492759Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.634044ms
level=error ts=2026-04-19T19:19:03.023536028Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-19T19:19:03.023546948Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-04-19T19:19:03.023935495Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-19T19:19:03.024167464Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:19:03.024421717Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-19T19:19:03.024437944Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-19T19:19:03.024435992Z
level=debug ts=2026-04-19T19:19:03.024454448Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-19T19:19:03.02447571Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-19T19:19:03.024535854Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-19T19:19:03.024542823Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-19T19:19:03.024541274Z
level=debug ts=2026-04-19T19:19:03.024549834Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-19T19:19:03.025776324Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T19:19:03.025853074Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.69 inf=eth0
ts=2026-04-19T19:19:03.026754071Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:19:03.026770963Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T19:19:03.042471414Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-19T19:19:03.042476498Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-04-19T19:19:03.0424765Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-19T19:19:03.042498741Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-04-19T19:19:03.042504174Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-19T19:19:03.042520226Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-19T19:19:03.04251992Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-04-19T19:19:03.042533384Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-19T19:19:03.042545551Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-19T19:19:03.042551681Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-19T19:19:03.042749027Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-04-19T19:19:03.042753209Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-19T19:19:03.042759775Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-04-19T19:19:03.042760783Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-04-19T19:19:03.042772331Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-04-19T19:19:03.042782515Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-19T19:19:03.042814016Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-19T19:19:03.042818478Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-19T19:19:03.042821531Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-19T19:19:03.042827311Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-19T19:19:03.042934143Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-19T19:19:03.042941325Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-19T19:19:03.04294641Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-19T19:19:03.042950965Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-04-19T19:19:03.042955371Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-19T19:19:03.042971793Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-19T19:19:03.042996319Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-19T19:19:03.043057978Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=87.296µs errors=false
level=info ts=2026-04-19T19:19:03.043065366Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-19T19:19:03.043201321Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=229.813µs errors=false
level=info ts=2026-04-19T19:19:03.043220069Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-19T19:19:03.04323348Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=262.532µs
level=info ts=2026-04-19T19:19:03.043257987Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-19T19:19:03.043295506Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-19T19:19:03.043298412Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=8.372848212s
level=info ts=2026-04-19T19:19:03.043334476Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-04-19T19:19:03.043364394Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-19T19:19:03.043375195Z caller=loki.go:599 msg="Loki started" startup_time=99.618269ms
level=debug ts=2026-04-19T19:19:03.043428361Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-19T19:19:03.043436684Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-04-19T19:19:03.043489081Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-19T19:19:04.312190713Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:19:04.31221327Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:19:08.043970552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:19:08.054386172Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:19:08.054405645Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:19:13.043816862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:19:14.862016966Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:19:14.86204077Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:19:18.043627741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:19:18.046214356Z caller=logging.go:137 msg="GET /ready (503) 90.253µs"
level=warn ts=2026-04-19T19:19:23.04340801Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:19:28.043752097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:19:28.044890811Z caller=logging.go:137 msg="GET /ready (503) 64.281µs"
ts=2026-04-19T19:19:30.546717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.65:7946"
ts=2026-04-19T19:19:30.548146968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.67:7946"
ts=2026-04-19T19:19:30.549441961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.68:7946"
level=info ts=2026-04-19T19:19:30.549856386Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=27.526302548s
level=info ts=2026-04-19T19:19:33.043738751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:19:33.043744922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:19:33.043774072Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-19T19:19:33.043781571Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:19:38.043491072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:19:39.09026128Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:19:43.04362391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:19:48.044174939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:19:50.809360577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:51992"
level=warn ts=2026-04-19T19:19:53.04397198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:19:56.308644551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:19:58.044304189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:20:02.783027318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=debug ts=2026-04-19T19:20:03.02334992Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:20:03.024396853Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:20:03.044326154Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:20:03.044345416Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:20:03.044353816Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:20:03.044392834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:20:08.043688606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:20:12.445474727Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-19T19:20:12.445537741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:20:12.455004297Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:20:13.044273832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:20:13.599023048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:20:18.043900789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:20:23.04421213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:20:28.044102872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:20:30.407252399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:20:30.413772946Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-19T19:20:32.784401564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=warn ts=2026-04-19T19:20:33.043616309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:20:33.043640193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:20:33.043660418Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:20:33.043668673Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:20:34.461972402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T19:20:37.537611714Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:20:38.043778648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:20:43.043870666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:20:48.043993358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:20:53.044304597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:20:55.549707932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:20:58.044079556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:21:02.785766824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:21:03.023539687Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:21:03.024586002Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-19T19:21:03.043706431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:21:03.043712385Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:21:03.043736003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:21:03.043743381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-19T19:21:05.254935279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:21:05.26254849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:21:08.043779338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:21:13.044167095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:21:13.273630101Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:21:18.043833613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:21:23.044399219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:21:27.417697566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T19:21:27.419246164Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:21:28.044012893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:21:32.787344933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:21:33.043402813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:21:33.0434237Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:21:33.043431691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:21:33.043430709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:21:38.043561266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:21:39.109333557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:47660"
level=warn ts=2026-04-19T19:21:43.043810522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:21:43.675184729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:21:48.043639587Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:21:53.043439743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:21:58.043456686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:21:59.078429596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:21:59.08494309Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-19T19:22:02.788449912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:22:02.910123983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42476"
level=debug ts=2026-04-19T19:22:03.023819761Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:22:03.024874551Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:22:03.043698669Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:22:03.043718332Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:22:03.043726601Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:22:03.043738566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:22:06.626777469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:22:08.044091844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:22:09.110834162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:43532"
level=warn ts=2026-04-19T19:22:13.043433316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:22:13.72928387Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:22:18.044413378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:22:23.043721922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:22:28.04374456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:22:32.790006515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:22:32.911859913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:33834"
level=info ts=2026-04-19T19:22:33.043371577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:22:33.043391471Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:22:33.043399875Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:22:33.043399135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:22:38.043986635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:22:39.111922145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:35638"
level=warn ts=2026-04-19T19:22:43.04352798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:22:47.152793111Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:22:48.04412964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:22:53.044338452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:22:53.778742032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:22:53.785965871Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:22:58.043614928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:23:02.790928847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=debug ts=2026-04-19T19:23:03.023672517Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:23:03.024739724Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:23:03.043954764Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:23:03.043971183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:23:03.043979523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:23:03.043993859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:23:08.043826203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:23:09.112791164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:55746"
level=warn ts=2026-04-19T19:23:13.043913963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:23:17.075463258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:23:18.043547608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:23:20.819686803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:37686"
level=warn ts=2026-04-19T19:23:23.043759874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:23:25.364780855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:23:28.043844912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:23:32.792494343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=warn ts=2026-04-19T19:23:33.04430536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:23:33.044305575Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:23:33.044343638Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:23:33.044351683Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:23:38.044238419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:23:40.408453187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:23:40.415981784Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:23:43.044161889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:23:47.738862946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:23:48.044088539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:23:53.043856714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:23:58.044453578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:24:02.793302786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:24:02.9152499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:54616"
level=info ts=2026-04-19T19:24:03.024572217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:24:03.025175024Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:24:03.043555397Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-19T19:24:03.043571318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:24:03.043584247Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:24:03.043604849Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:24:03.043613739Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-19T19:24:03.043679077Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-04-19T19:24:03.043858151Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-19T19:24:07.39958682Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=warn ts=2026-04-19T19:24:08.044002379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:11.754616071Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T19:24:13.043557803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:13.735706102Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:24:16.109139478Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-19T19:24:18.044017456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:20.463249182Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-19T19:24:23.044291254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:24.818988201Z caller=checkpoint.go:378 msg="writing series" size="525 kB"
level=warn ts=2026-04-19T19:24:28.044440264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:29.173536733Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-04-19T19:24:30.089997902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:24:30.096579932Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-19T19:24:32.794464794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:24:33.044162466Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:24:33.044184366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:24:33.044192673Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:24:33.044209883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:33.528588657Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-04-19T19:24:34.771831782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T19:24:37.883605972Z caller=checkpoint.go:378 msg="writing series" size="511 B"
level=warn ts=2026-04-19T19:24:38.04380466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:39.412872327Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:24:42.238412282Z caller=checkpoint.go:378 msg="writing series" size="308 kB"
level=warn ts=2026-04-19T19:24:43.043846461Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:46.592657946Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-19T19:24:48.0441436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:50.947620381Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-19T19:24:53.043854367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:55.302761927Z caller=checkpoint.go:378 msg="writing series" size="315 kB"
level=debug ts=2026-04-19T19:24:55.302787357Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=13
level=warn ts=2026-04-19T19:24:58.043817759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:24:59.657129276Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
ts=2026-04-19T19:25:02.795599719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:25:03.024442541Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:25:03.024448489Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=warn ts=2026-04-19T19:25:03.043776056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:25:03.043793787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:25:03.043808207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:25:03.043815721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:25:04.012285604Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-19T19:25:08.043797842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:08.367085552Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=debug ts=2026-04-19T19:25:12.72226908Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=warn ts=2026-04-19T19:25:13.043719709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:15.334539501Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:25:17.080432395Z caller=checkpoint.go:378 msg="writing series" size="4.3 MB"
level=debug ts=2026-04-19T19:25:17.08045741Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.6 MB" series=5
level=warn ts=2026-04-19T19:25:18.044046259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:21.4320139Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-04-19T19:25:22.969160515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:25:22.97962676Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:25:23.043619148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:25.786690678Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-04-19T19:25:28.043593459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:30.14090611Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
ts=2026-04-19T19:25:32.796700537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
ts=2026-04-19T19:25:32.918784249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:58110"
level=info ts=2026-04-19T19:25:33.043522319Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:25:33.043547515Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:25:33.043555807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:25:33.043549128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:34.496296684Z caller=checkpoint.go:378 msg="writing series" size="155 kB"
level=warn ts=2026-04-19T19:25:38.04370985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:38.85089761Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=debug ts=2026-04-19T19:25:39.74026634Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:25:43.043843298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:43.206194429Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-19T19:25:45.599102051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T19:25:47.560426981Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=warn ts=2026-04-19T19:25:48.044161298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:25:50.82630168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:48806"
level=debug ts=2026-04-19T19:25:51.915578182Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-19T19:25:53.043888671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:25:56.270041284Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=debug ts=2026-04-19T19:25:56.27006692Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-19T19:25:58.043899659Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:00.624983004Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
ts=2026-04-19T19:26:02.797710114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:26:02.920245335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:36722"
level=info ts=2026-04-19T19:26:03.024230542Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:26:03.024522905Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:26:03.043566574Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:26:03.043584301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:26:03.043593281Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:26:03.043637418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:26:03.97387627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:26:03.983299652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:26:04.180004542Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:26:04.980111809Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-19T19:26:08.043832456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:09.334811995Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=warn ts=2026-04-19T19:26:13.043429514Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:13.689625882Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=warn ts=2026-04-19T19:26:18.044031061Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:18.044086328Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-04-19T19:26:22.400308015Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-19T19:26:22.400345715Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=6
level=warn ts=2026-04-19T19:26:23.044230331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:26.754357188Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=warn ts=2026-04-19T19:26:28.043968647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:31.108815579Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
ts=2026-04-19T19:26:32.79927209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:26:33.043358642Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:26:33.043383798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:26:33.04339172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:26:33.043419833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:35.464205514Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-19T19:26:38.043569122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:38.107378802Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:26:39.819032358Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=warn ts=2026-04-19T19:26:43.044049346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:44.173278615Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-19T19:26:48.044243352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:48.528131522Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-04-19T19:26:52.882713339Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-19T19:26:53.044086525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:57.238091315Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=warn ts=2026-04-19T19:26:58.043873891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:26:58.889177293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T19:26:58.890784215Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T19:26:59.340239723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:26:59.347222008Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:26:59.347247439Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:26:59.347277979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:26:59.353865814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:27:01.592702149Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
ts=2026-04-19T19:27:02.800042281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:27:03.024007621Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:27:03.025053655Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:27:03.044143042Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:27:03.044163886Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:27:03.044174713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:27:03.044167473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:05.947891659Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-19T19:27:08.044014198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:27:09.122450245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:32990"
level=debug ts=2026-04-19T19:27:09.302402614Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:27:10.301997163Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-19T19:27:12.989363789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:27:13.000375051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:27:13.043422722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:14.657104516Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-19T19:27:18.044170954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:19.012649155Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-19T19:27:23.044373951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:23.366920341Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=debug ts=2026-04-19T19:27:27.721764141Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-19T19:27:28.04427608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:32.076167646Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-04-19T19:27:32.801213916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:27:32.924240199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:40388"
level=info ts=2026-04-19T19:27:33.043464619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:27:33.043486671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:33.043495336Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:27:33.043505617Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:27:36.130357332Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:27:36.431302289Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-19T19:27:38.044171515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:40.785882055Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-19T19:27:43.04382553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:27:44.851640638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:27:44.860837567Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:27:45.140684592Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=debug ts=2026-04-19T19:27:45.140708204Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=19
level=warn ts=2026-04-19T19:27:48.0435872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:49.495984004Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
ts=2026-04-19T19:27:50.831135329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:45416"
level=warn ts=2026-04-19T19:27:53.044195833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:53.253926357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:27:53.850536011Z caller=checkpoint.go:378 msg="writing series" size="307 kB"
level=warn ts=2026-04-19T19:27:58.044408967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:27:58.206197309Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-19T19:28:02.560088988Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
ts=2026-04-19T19:28:02.802772212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:28:02.925579455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:44744"
level=debug ts=2026-04-19T19:28:03.023664774Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:28:03.024707498Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:28:03.044389531Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:28:03.044407858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:28:03.044415843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:28:03.044416823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:28:06.451817Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:28:06.915828839Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-19T19:28:08.044080645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:28:09.124107927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:60686"
level=debug ts=2026-04-19T19:28:11.269892595Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-19T19:28:13.044460862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:28:15.625371321Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
level=warn ts=2026-04-19T19:28:18.044057822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:28:19.980295538Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
ts=2026-04-19T19:28:20.831835987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53792"
level=warn ts=2026-04-19T19:28:23.043592013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:28:24.334245201Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-19T19:28:28.044022598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:28:28.689530906Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=info ts=2026-04-19T19:28:30.109300075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:28:30.116734914Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:28:31.706113099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:28:32.804284786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:28:33.043376974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:28:33.043398257Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:28:33.043405563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:28:33.043406351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:28:33.044415805Z caller=checkpoint.go:390 msg="flushing series" totalSize="607 kB" series=10
level=info ts=2026-04-19T19:28:33.04973949Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-19T19:28:33.053044366Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009233895s
level=debug ts=2026-04-19T19:28:35.570897781Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:28:38.043814863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:28:39.125553092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45930"
level=warn ts=2026-04-19T19:28:43.043433927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:28:48.043826193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:28:50.832762445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:55018"
level=warn ts=2026-04-19T19:28:53.04407398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:28:58.044047201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:29:02.805671572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
ts=2026-04-19T19:29:02.92866719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:39584"
level=debug ts=2026-04-19T19:29:03.02411305Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:29:03.025166873Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:29:03.043494483Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-19T19:29:03.043536129Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:29:03.043554962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:29:03.043562887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:29:03.043560037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:29:03.043627443Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-04-19T19:29:03.043822194Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-19T19:29:03.613213125Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:29:07.398759767Z caller=checkpoint.go:378 msg="writing series" size="1.6 kB"
level=warn ts=2026-04-19T19:29:08.044307101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:29:09.127017786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58874"
level=info ts=2026-04-19T19:29:11.695941645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:29:11.70409591Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:29:11.754244103Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T19:29:13.043846408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:29:16.109350692Z caller=checkpoint.go:378 msg="writing series" size="9.6 kB"
level=warn ts=2026-04-19T19:29:18.043872951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:29:20.463471387Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-19T19:29:21.772988108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T19:29:23.043411642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:29:24.818901832Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
level=warn ts=2026-04-19T19:29:28.044053763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:29:29.173244207Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=debug ts=2026-04-19T19:29:32.619478165Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:29:32.807008503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:29:33.043976423Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:29:33.043979015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:29:33.043997351Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:29:33.044006547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:29:33.528370622Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=debug ts=2026-04-19T19:29:37.883238881Z caller=checkpoint.go:378 msg="writing series" size="511 B"
level=warn ts=2026-04-19T19:29:38.044358083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:29:42.237929367Z caller=checkpoint.go:378 msg="writing series" size="308 kB"
level=warn ts=2026-04-19T19:29:43.044277043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:29:46.592553983Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-19T19:29:48.043756569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:29:50.835308298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57700"
level=debug ts=2026-04-19T19:29:50.948053621Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-19T19:29:53.044428203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:29:54.077158978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:29:54.086360055Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:29:55.302680877Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=debug ts=2026-04-19T19:29:55.302705717Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=13
level=warn ts=2026-04-19T19:29:58.043694274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:29:59.656853998Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=debug ts=2026-04-19T19:30:01.991424174Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:30:02.807991129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=debug ts=2026-04-19T19:30:03.024191838Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:30:03.024221084Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:30:03.025561921Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-19T19:30:03.025622801Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974029 n=1
level=debug ts=2026-04-19T19:30:03.025684062Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974029 n=1
level=warn ts=2026-04-19T19:30:03.044340192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:30:03.044340041Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:30:03.044365376Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:30:03.04437624Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:30:04.012296746Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-19T19:30:08.044447686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:30:08.367052323Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=debug ts=2026-04-19T19:30:10.122868002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:30:12.72180137Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=warn ts=2026-04-19T19:30:13.043579404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:30:17.079760668Z caller=checkpoint.go:378 msg="writing series" size="5.0 MB"
level=debug ts=2026-04-19T19:30:17.079786208Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.3 MB" series=5
level=warn ts=2026-04-19T19:30:18.043901962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:30:21.431846526Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=warn ts=2026-04-19T19:30:23.044467774Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:30:25.785818125Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-19T19:30:28.044430745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:30:30.141415291Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
ts=2026-04-19T19:30:32.809828322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:30:32.932386539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:48496"
level=warn ts=2026-04-19T19:30:33.044310912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:30:33.044327817Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:30:33.044347232Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:30:33.044355721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-19T19:30:34.124296908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:30:34.131723778Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:30:34.495637608Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=debug ts=2026-04-19T19:30:37.447410652Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:30:38.043452043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:30:38.850845441Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
ts=2026-04-19T19:30:39.130308189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:51574"
level=warn ts=2026-04-19T19:30:43.043746709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:30:43.206042017Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-19T19:30:47.560502326Z caller=checkpoint.go:378 msg="writing series" size="310 kB"
level=warn ts=2026-04-19T19:30:48.043647007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:30:50.837568476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:42960"
level=debug ts=2026-04-19T19:30:51.910701881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:30:51.912256733Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T19:30:51.915416755Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=debug ts=2026-04-19T19:30:51.915427852Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-19T19:30:53.044081664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:30:56.270080158Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-19T19:30:58.044104868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:00.625341828Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
ts=2026-04-19T19:31:02.811022298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:31:03.024333314Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:31:03.024337502Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:31:03.04422247Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:31:03.044238301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:31:03.044246137Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:31:03.044253017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:04.980298471Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-19T19:31:08.044270481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:31:09.131862573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:55814"
level=debug ts=2026-04-19T19:31:09.334770612Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=debug ts=2026-04-19T19:31:10.497926587Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:31:13.044044927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:13.689073057Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-19T19:31:16.809045792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:31:16.818474893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:31:18.044260858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:18.044283595Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-04-19T19:31:22.400335063Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-19T19:31:22.400359382Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=7
level=warn ts=2026-04-19T19:31:23.043821054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:26.753569323Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-04-19T19:31:28.043606374Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:31.108543124Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
ts=2026-04-19T19:31:32.812082159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=warn ts=2026-04-19T19:31:33.043429248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:31:33.043427972Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:31:33.04345334Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:31:33.043467033Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:31:35.464165544Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-19T19:31:38.043522409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:39.819102882Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-04-19T19:31:43.043990017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:43.384750912Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:31:44.173688548Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-19T19:31:48.043732623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:48.354254969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:31:48.528507241Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-04-19T19:31:52.883466173Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-19T19:31:53.044133553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:31:57.238403342Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=warn ts=2026-04-19T19:31:58.043743263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:01.592907012Z caller=checkpoint.go:378 msg="writing series" size="296 kB"
ts=2026-04-19T19:32:02.81323002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:32:03.02433082Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:32:03.024735952Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:32:03.043368069Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:32:03.043386774Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:32:03.043394195Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:32:03.043394973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:32:03.328672898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:32:03.335652425Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:32:05.947107308Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-19T19:32:08.043653043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:08.162207044Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:32:10.30264417Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-19T19:32:13.044012609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:14.65683006Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-19T19:32:18.043894892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:19.01213479Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-19T19:32:20.840270627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:58326"
level=warn ts=2026-04-19T19:32:23.043490709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:23.367372168Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=debug ts=2026-04-19T19:32:23.367396701Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-04-19T19:32:27.721827028Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-19T19:32:28.043977299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:32.076434519Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-04-19T19:32:32.81481158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:32:32.93812053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:51260"
level=info ts=2026-04-19T19:32:33.043651178Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:32:33.0436716Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:32:33.043680022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:32:33.04369186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:34.321410768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:32:36.431073141Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-19T19:32:38.043710265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:38.353119Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:32:40.786303243Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-19T19:32:43.043707403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:45.141446804Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=warn ts=2026-04-19T19:32:48.044056037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:49.496193137Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
ts=2026-04-19T19:32:50.841411843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:46830"
level=warn ts=2026-04-19T19:32:53.043418915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:53.850963378Z caller=checkpoint.go:378 msg="writing series" size="444 kB"
level=info ts=2026-04-19T19:32:56.541693503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:32:56.549180111Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:32:56.549200188Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:32:56.549222782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:32:56.555262016Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:32:58.044348853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:32:58.206167037Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-19T19:33:02.56047361Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=debug ts=2026-04-19T19:33:02.560495173Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
ts=2026-04-19T19:33:02.816102316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:33:03.023993835Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:33:03.025042975Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:33:03.044357439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:33:03.044358051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:33:03.044373934Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:33:03.044382113Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:33:06.478118336Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:33:06.915247806Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-19T19:33:08.043692201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:33:10.3062778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:33:10.315442896Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:33:11.270053837Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-19T19:33:13.043596056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:33:15.625282607Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-19T19:33:18.043638254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:33:19.980173425Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-19T19:33:23.043836218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:33:24.334847196Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-19T19:33:28.044356422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:33:28.689857878Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
ts=2026-04-19T19:33:32.817146045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:33:32.940715556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:54542"
level=info ts=2026-04-19T19:33:33.043369666Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:33:33.043387274Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:33:33.043394786Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:33:33.043397162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:33:33.044409221Z caller=checkpoint.go:390 msg="flushing series" totalSize="64 kB" series=6
level=info ts=2026-04-19T19:33:33.050012036Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-19T19:33:33.051193486Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007411093s
level=info ts=2026-04-19T19:33:33.918504397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:33:33.925388625Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:33:34.346176124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T19:33:38.043451099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:33:38.227652299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:33:43.043662099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:33:48.043720822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:33:50.843827627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:34218"
level=warn ts=2026-04-19T19:33:53.044013693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:33:58.047246424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:34:02.818381553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:34:03.024234424Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:34:03.024241616Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:34:03.043652385Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-19T19:34:03.043669309Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:34:03.043686252Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:34:03.043696042Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:34:03.043724294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:34:03.043776604Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-19T19:34:03.043968741Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-19T19:34:03.168093266Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:34:07.398860833Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=warn ts=2026-04-19T19:34:08.04356507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:11.753784752Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T19:34:13.04377982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:16.109037199Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-19T19:34:18.038812513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="21 MB" avg_uncomp="21 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:34:18.043639818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-04-19T19:34:18.049787489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:34:20.463680122Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-19T19:34:23.043442006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:24.818515614Z caller=checkpoint.go:378 msg="writing series" size="397 kB"
level=warn ts=2026-04-19T19:34:28.043849039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:29.173370883Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=debug ts=2026-04-19T19:34:29.975994263Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:34:31.172309754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:34:32.819237103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:34:33.044183666Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:34:33.044209655Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:34:33.044218648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:34:33.044213694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:33.528879208Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=debug ts=2026-04-19T19:34:37.883009107Z caller=checkpoint.go:378 msg="writing series" size="511 B"
level=warn ts=2026-04-19T19:34:38.043942972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:42.237922021Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=warn ts=2026-04-19T19:34:43.04418003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:46.592935482Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-19T19:34:48.043684286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:50.947782249Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-19T19:34:53.043683264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:55.303140753Z caller=checkpoint.go:378 msg="writing series" size="273 kB"
level=debug ts=2026-04-19T19:34:55.303165983Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-04-19T19:34:58.043804704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:34:59.657808206Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=debug ts=2026-04-19T19:35:01.924564321Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:35:02.821403932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:35:03.024471151Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:35:03.025287021Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:35:03.044253512Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:35:03.04427551Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:35:03.044283229Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:35:03.044307415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:04.011943663Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-19T19:35:07.365046437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:35:07.366981576Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:35:08.043451923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:08.367555686Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=debug ts=2026-04-19T19:35:12.72192298Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=warn ts=2026-04-19T19:35:13.044369314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:35:15.470747714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:35:15.479687077Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:35:17.079774984Z caller=checkpoint.go:378 msg="writing series" size="5.5 MB"
level=debug ts=2026-04-19T19:35:17.079798088Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.8 MB" series=5
level=warn ts=2026-04-19T19:35:18.043751156Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:35:20.848510994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:60524"
level=debug ts=2026-04-19T19:35:21.431812382Z caller=checkpoint.go:378 msg="writing series" size="160 kB"
level=warn ts=2026-04-19T19:35:23.043423624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:25.786580599Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=warn ts=2026-04-19T19:35:28.044186752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:30.1418722Z caller=checkpoint.go:378 msg="writing series" size="277 kB"
level=debug ts=2026-04-19T19:35:31.682065531Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:35:32.822476175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:35:33.044235335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:35:33.044258808Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:35:33.044266963Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:35:33.044264337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:34.496504949Z caller=checkpoint.go:378 msg="writing series" size="219 kB"
level=warn ts=2026-04-19T19:35:38.044238652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:38.85087863Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=warn ts=2026-04-19T19:35:43.043866932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:43.206226052Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-19T19:35:46.576698277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:35:47.56165477Z caller=checkpoint.go:378 msg="writing series" size="2.8 MB"
level=debug ts=2026-04-19T19:35:47.561677883Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.5 MB" series=7
level=warn ts=2026-04-19T19:35:48.044011348Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:35:50.850263293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:33268"
level=debug ts=2026-04-19T19:35:51.91542223Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=warn ts=2026-04-19T19:35:53.043895495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:56.270078562Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-19T19:35:58.044028588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:35:59.410377794Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:36:00.625055963Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=info ts=2026-04-19T19:36:01.306391204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:36:01.320116576Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-19T19:36:02.824251543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:36:03.023903924Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:36:03.024990423Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-19T19:36:03.043911505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:36:03.043912542Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:36:03.04393905Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:36:03.043952656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:36:04.979765811Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-19T19:36:08.043904103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:09.334827283Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=warn ts=2026-04-19T19:36:13.043871214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:13.689756885Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=warn ts=2026-04-19T19:36:18.044239167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:18.044255805Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
ts=2026-04-19T19:36:20.851740529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:58560"
level=debug ts=2026-04-19T19:36:22.39994492Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-19T19:36:22.399969302Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=8
level=warn ts=2026-04-19T19:36:23.044428593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:24.155411477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:36:26.754353107Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=warn ts=2026-04-19T19:36:28.043868169Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:31.108549576Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-04-19T19:36:32.826198442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:36:33.043487325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:36:33.043514396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:36:33.043524013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:36:33.043563863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:35.381700731Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:36:35.464115248Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-19T19:36:38.043849722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:36:39.146364349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58704"
level=debug ts=2026-04-19T19:36:39.818882743Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-19T19:36:43.044382179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:44.173951812Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-19T19:36:44.214093469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:36:44.223794708Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:36:48.044110291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:48.528151963Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=debug ts=2026-04-19T19:36:52.883655588Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=warn ts=2026-04-19T19:36:53.044259592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:36:57.237778871Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=warn ts=2026-04-19T19:36:58.043989677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:01.450100329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:37:01.593576416Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
ts=2026-04-19T19:37:02.827498115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:37:03.024426756Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:37:03.024886003Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:37:03.043677515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:37:03.04368471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:03.043703949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:37:03.043725774Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:37:05.94770265Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-19T19:37:08.044227931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:37:09.147926367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:48626"
level=debug ts=2026-04-19T19:37:10.302409603Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-19T19:37:13.043619679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:14.657434953Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=debug ts=2026-04-19T19:37:14.657460026Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-19T19:37:18.04442417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:19.01234839Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-19T19:37:23.043438735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:23.367424529Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=debug ts=2026-04-19T19:37:26.291677635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:37:27.721672144Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=warn ts=2026-04-19T19:37:28.044468151Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:28.347857185Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-19T19:37:28.347933962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:37:28.358308368Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:37:32.076885688Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-04-19T19:37:32.829227632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:37:32.870052172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:37:32.878413204Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-19T19:37:33.04342313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:37:33.04345384Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:37:33.043464056Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:37:33.043457842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:36.431514333Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-19T19:37:38.044230644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:37:39.149746302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:34056"
level=debug ts=2026-04-19T19:37:40.786622904Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-19T19:37:43.044273001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:45.140901004Z caller=checkpoint.go:378 msg="writing series" size="179 kB"
level=info ts=2026-04-19T19:37:45.609406599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:37:45.623467069Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:37:48.044216176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:49.496580338Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-19T19:37:53.044250612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:53.850994918Z caller=checkpoint.go:378 msg="writing series" size="491 kB"
level=debug ts=2026-04-19T19:37:53.851019847Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-04-19T19:37:58.043774305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:37:58.205634403Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-19T19:37:59.836765256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:38:02.561009742Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
ts=2026-04-19T19:38:02.830995038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:38:03.024621021Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:38:03.025099523Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:38:03.043986689Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:38:03.044007434Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-19T19:38:03.043997523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:38:03.044019544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:38:06.915887183Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-19T19:38:08.044257804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:38:09.151030659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:56532"
level=debug ts=2026-04-19T19:38:11.27020786Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=warn ts=2026-04-19T19:38:13.04383257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:38:15.62512477Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-19T19:38:17.214245386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:38:17.22250275Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:38:18.04398922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:38:18.875397936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:38:19.979720456Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-19T19:38:23.044051177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:38:24.226291101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:38:24.234490699Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:38:24.334730033Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-19T19:38:27.224871224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:38:28.043919156Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:38:28.689689751Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
ts=2026-04-19T19:38:32.832496352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=warn ts=2026-04-19T19:38:33.043431566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:38:33.043441802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:38:33.043464998Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:38:33.043475595Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:38:33.044532622Z caller=checkpoint.go:390 msg="flushing series" totalSize="516 kB" series=8
level=info ts=2026-04-19T19:38:33.10641392Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-19T19:38:33.107909534Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.063982924s
level=warn ts=2026-04-19T19:38:38.043845126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:38:43.043738866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:38:48.044413087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:38:53.043550876Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:38:58.044349485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:02.777419809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:39:02.834012683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:39:03.024383503Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:39:03.02439827Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:39:03.044174441Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-19T19:39:03.044182321Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-19T19:39:03.044192355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:39:03.044201915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:39:03.044226228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:39:03.044300506Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-04-19T19:39:03.044478074Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-19T19:39:07.135854108Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-04-19T19:39:07.164953979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:39:07.17481238Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:39:07.174833136Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:39:07.174855369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:39:07.181071558Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:39:08.044962384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:39:09.153926151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:52826"
level=debug ts=2026-04-19T19:39:11.227050772Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T19:39:13.043677354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:15.318205977Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-19T19:39:15.584388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:39:15.586237012Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:39:18.04425593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:19.408793824Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-19T19:39:21.152890678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:39:21.16510493Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:39:23.044064545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:23.499865748Z caller=checkpoint.go:378 msg="writing series" size="518 kB"
level=info ts=2026-04-19T19:39:23.640355299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:39:23.646969842Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:39:27.590127453Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-19T19:39:28.043611418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:28.439568336Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:39:31.681759068Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
ts=2026-04-19T19:39:32.835250899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:39:33.043667208Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:39:33.043693144Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:39:33.043702031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:39:33.043699248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:35.772528214Z caller=checkpoint.go:378 msg="writing series" size="511 B"
level=warn ts=2026-04-19T19:39:38.043813482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:39.863767679Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=warn ts=2026-04-19T19:39:43.043395702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:43.954572445Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-19T19:39:48.044261255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:48.044576517Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-19T19:39:50.906686312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:39:50.915829104Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:39:52.136388046Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=debug ts=2026-04-19T19:39:52.136411744Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-04-19T19:39:53.043403347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:39:54.246392693Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:39:56.227431946Z caller=checkpoint.go:378 msg="writing series" size="248 kB"
level=debug ts=2026-04-19T19:39:56.334473217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T19:39:58.044007609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:00.317254584Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-19T19:40:02.836245802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:40:02.959151363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42844"
level=debug ts=2026-04-19T19:40:03.023417669Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:40:03.024459774Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-19T19:40:03.043781709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:40:03.043785645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:40:03.043811237Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:40:03.043819311Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:40:04.408303163Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-19T19:40:08.043934951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:08.499229603Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
ts=2026-04-19T19:40:09.156687768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:42506"
level=debug ts=2026-04-19T19:40:12.595964436Z caller=checkpoint.go:378 msg="writing series" size="5.9 MB"
level=debug ts=2026-04-19T19:40:12.595987722Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.2 MB" series=5
level=warn ts=2026-04-19T19:40:13.044358408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:16.681338569Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-04-19T19:40:18.043828487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:18.496835798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-19T19:40:19.845823708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:40:19.854733478Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:40:20.772181269Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
ts=2026-04-19T19:40:20.861417198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:38772"
level=warn ts=2026-04-19T19:40:23.043820243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:24.863560174Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=warn ts=2026-04-19T19:40:28.043549806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:28.954333751Z caller=checkpoint.go:378 msg="writing series" size="255 kB"
ts=2026-04-19T19:40:32.837739911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:40:33.044014034Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:40:33.044035676Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:40:33.044043227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:40:33.044040886Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:33.045275491Z caller=checkpoint.go:378 msg="writing series" size="252 kB"
level=debug ts=2026-04-19T19:40:37.135676704Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-04-19T19:40:38.043695464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:41.227292142Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-19T19:40:43.044439522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:40:43.149713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:40:43.15725344Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:40:45.320731099Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB"
level=debug ts=2026-04-19T19:40:45.320755083Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.4 MB" series=8
level=warn ts=2026-04-19T19:40:48.043654691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:48.645172752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:40:49.344270249Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:40:49.408441354Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
ts=2026-04-19T19:40:50.862543716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:59826"
level=warn ts=2026-04-19T19:40:53.043498422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:40:53.499405767Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=debug ts=2026-04-19T19:40:57.590296154Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-19T19:40:58.0444272Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:01.681532573Z caller=checkpoint.go:378 msg="writing series" size="325 kB"
ts=2026-04-19T19:41:02.838523578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:41:03.024223217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:41:03.024552624Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:41:03.04384964Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:41:03.0438689Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:41:03.043877404Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:41:03.043872233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:05.771902463Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-19T19:41:08.043566368Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:41:09.394964704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:41:09.405835667Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:41:09.863205259Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-19T19:41:13.043553805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:13.954489931Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=warn ts=2026-04-19T19:41:18.044129789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:18.045427332Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=debug ts=2026-04-19T19:41:18.045441899Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-04-19T19:41:18.353745871Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:41:20.863724915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53870"
level=debug ts=2026-04-19T19:41:22.136381379Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=warn ts=2026-04-19T19:41:23.043684658Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:26.228267242Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-19T19:41:26.228300075Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=2
level=warn ts=2026-04-19T19:41:28.044186329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:30.317557948Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
ts=2026-04-19T19:41:32.840328135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:41:32.962148531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:59908"
level=info ts=2026-04-19T19:41:33.043784525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:41:33.043816862Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:41:33.043828901Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:41:33.043821884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:34.408910796Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-19T19:41:38.044124019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:38.499820537Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
ts=2026-04-19T19:41:39.160147301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:52432"
level=info ts=2026-04-19T19:41:41.735472046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:41:41.745048821Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:41:42.590414705Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-19T19:41:43.04435633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:46.681604109Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-19T19:41:48.04392114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:50.253102615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:41:50.771968111Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=warn ts=2026-04-19T19:41:53.044132618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:54.019597967Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:41:54.862954367Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-04-19T19:41:58.044008755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:41:58.95437871Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
ts=2026-04-19T19:42:02.841821054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:42:03.02423076Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:42:03.024550621Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:42:03.044157284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:42:03.044179592Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:42:03.044188872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:42:03.044236041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:03.04545507Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=info ts=2026-04-19T19:42:04.548847627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:42:04.556572294Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:42:07.13580705Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=debug ts=2026-04-19T19:42:07.13583242Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-19T19:42:08.044006917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:11.227021117Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-19T19:42:13.04377124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:15.318285894Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-19T19:42:18.044296797Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:19.408218036Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-19T19:42:23.043634368Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:23.499950838Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=info ts=2026-04-19T19:42:27.10789419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:42:27.115695823Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:42:27.590936632Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=warn ts=2026-04-19T19:42:28.04355266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:29.233109039Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:42:31.681872254Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-04-19T19:42:32.84326989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:42:33.043625808Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:42:33.0436537Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-19T19:42:33.043653577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:42:33.043665384Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:42:35.772393607Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-19T19:42:38.044086752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:39.571480001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:42:39.863725339Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-19T19:42:43.043877604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:43.953911953Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=warn ts=2026-04-19T19:42:48.043422479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:42:48.045230381Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-19T19:42:52.13606613Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-19T19:42:53.043550098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:42:55.527237608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:42:55.538694433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:42:56.226868525Z caller=checkpoint.go:378 msg="writing series" size="463 kB"
level=debug ts=2026-04-19T19:42:56.226894679Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=12
level=debug ts=2026-04-19T19:42:56.691367667Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:42:58.043664678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:43:00.318102036Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-19T19:43:02.84453105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=debug ts=2026-04-19T19:43:03.023849265Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:43:03.024936879Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-19T19:43:03.043471701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:43:03.043478269Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:43:03.043498536Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:43:03.043506383Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:43:04.408744867Z caller=checkpoint.go:378 msg="writing series" size="331 kB"
level=warn ts=2026-04-19T19:43:08.044308217Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:43:08.499465376Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
ts=2026-04-19T19:43:09.163955449Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:54214"
level=debug ts=2026-04-19T19:43:12.590012701Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-19T19:43:13.044287685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:43:13.844859683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:43:13.853899134Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:43:16.681075948Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-19T19:43:17.409834481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:43:17.411729369Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:43:18.04442725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:43:20.772258093Z caller=checkpoint.go:378 msg="writing series" size="450 kB"
level=debug ts=2026-04-19T19:43:22.644998035Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:43:23.043806223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:43:24.863359004Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-19T19:43:28.043895352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:43:28.953887976Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
ts=2026-04-19T19:43:32.845977853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:43:32.967564025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:59072"
level=info ts=2026-04-19T19:43:33.043413777Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:43:33.043427761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:43:33.043447289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:43:33.043458523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:43:33.044492458Z caller=checkpoint.go:390 msg="flushing series" totalSize="953 kB" series=8
level=info ts=2026-04-19T19:43:33.052244938Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-19T19:43:33.056039926Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.011602282s
level=info ts=2026-04-19T19:43:36.861486611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:43:36.870913675Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:43:38.043886604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:43:43.043458243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:43:48.044163675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:43:53.043989455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:43:53.921123924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:43:53.928660562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:43:55.775709667Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:43:58.04443961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:44:02.847087935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=debug ts=2026-04-19T19:44:03.02381708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:44:03.02486908Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:44:03.044160433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-19T19:44:03.044169682Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-19T19:44:03.044179242Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:44:03.044187576Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:44:03.044240933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:44:03.044284332Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-04-19T19:44:03.044455207Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-19T19:44:07.135785172Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=warn ts=2026-04-19T19:44:08.044389519Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:10.36541143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:44:11.227054664Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T19:44:13.043843591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:15.317295797Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=warn ts=2026-04-19T19:44:18.044010575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:19.408171601Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-19T19:44:20.87280039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:43998"
level=debug ts=2026-04-19T19:44:22.592787878Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:44:23.043782915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:23.49996356Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
level=debug ts=2026-04-19T19:44:27.590900571Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-19T19:44:28.043684569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:31.681903669Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=info ts=2026-04-19T19:44:32.118283001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:44:32.127670897Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:44:32.127691811Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:44:32.127717878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:44:32.134149085Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-19T19:44:32.848273472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:44:33.043552632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:44:33.043585454Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:44:33.043598282Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:44:33.043590636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:44:34.923774771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:44:34.931040853Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:44:35.772301307Z caller=checkpoint.go:378 msg="writing series" size="511 B"
level=warn ts=2026-04-19T19:44:38.043642529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:39.863756672Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=warn ts=2026-04-19T19:44:43.043770085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:43.954384808Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=info ts=2026-04-19T19:44:45.809304986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:44:45.818805947Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:44:48.043849317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:48.044728929Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-19T19:44:50.540217316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:44:52.136468258Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
level=debug ts=2026-04-19T19:44:52.136497829Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=13
level=debug ts=2026-04-19T19:44:52.399676008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T19:44:53.044229925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:44:56.227415456Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=warn ts=2026-04-19T19:44:58.044161496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:00.318263386Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-19T19:45:02.850013093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:45:03.02450293Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:45:03.024909426Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=debug ts=2026-04-19T19:45:03.026182363Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-19T19:45:03.026221526Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974030 n=1
level=debug ts=2026-04-19T19:45:03.026273261Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974030 n=1
level=info ts=2026-04-19T19:45:03.04385138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:45:03.043854031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:03.043871615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:45:03.043882022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:45:04.408936717Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-19T19:45:08.043873396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:08.499289244Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
ts=2026-04-19T19:45:09.169864676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:41734"
level=debug ts=2026-04-19T19:45:12.594952993Z caller=checkpoint.go:378 msg="writing series" size="6.4 MB"
level=debug ts=2026-04-19T19:45:12.594985059Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.6 MB" series=5
level=warn ts=2026-04-19T19:45:13.043817261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:16.681810984Z caller=checkpoint.go:378 msg="writing series" size="331 kB"
level=debug ts=2026-04-19T19:45:16.691862227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:45:18.043632158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:20.772659465Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=info ts=2026-04-19T19:45:22.521792877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:45:22.529668174Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:45:23.044367572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:24.86329726Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=warn ts=2026-04-19T19:45:28.044061263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:28.250186152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:45:28.954444085Z caller=checkpoint.go:378 msg="writing series" size="421 kB"
level=info ts=2026-04-19T19:45:29.327576269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:45:29.334496985Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:45:29.334517319Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:45:29.334543189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:45:29.340956457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-19T19:45:32.851859093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:45:33.043891015Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:45:33.043894697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:33.043916717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:45:33.043925273Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:45:33.045046804Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=debug ts=2026-04-19T19:45:37.136092528Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=debug ts=2026-04-19T19:45:37.136120395Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-19T19:45:38.046866486Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:41.227048798Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-19T19:45:43.043572022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:45.321483072Z caller=checkpoint.go:378 msg="writing series" size="7.6 MB"
level=debug ts=2026-04-19T19:45:45.321514718Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.6 MB" series=2
level=warn ts=2026-04-19T19:45:48.043523852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:45:48.299901866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:45:48.309211707Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:45:49.408641279Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=debug ts=2026-04-19T19:45:50.777305811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:45:53.04413304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:45:53.499553166Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-19T19:45:57.590907553Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-19T19:45:58.043571307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:01.681612517Z caller=checkpoint.go:378 msg="writing series" size="363 kB"
ts=2026-04-19T19:46:02.853644072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:46:03.024208852Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:46:03.02458629Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:46:03.043980622Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:46:03.04399601Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:46:03.044003475Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:46:03.044017441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:05.772546364Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-19T19:46:08.044145663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:09.864934548Z caller=checkpoint.go:378 msg="writing series" size="3.1 MB"
level=debug ts=2026-04-19T19:46:09.864957413Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.0 MB" series=6
level=warn ts=2026-04-19T19:46:13.044213697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:13.954117669Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-04-19T19:46:14.049545464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:46:14.059400988Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:46:18.043715791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:18.044850695Z caller=checkpoint.go:378 msg="writing series" size="443 kB"
ts=2026-04-19T19:46:20.878024531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:37444"
level=debug ts=2026-04-19T19:46:22.135765179Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-04-19T19:46:22.179808583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:46:22.186431992Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:46:23.044334607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:24.227731519Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:46:26.227165523Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-19T19:46:26.227190975Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=4
level=warn ts=2026-04-19T19:46:28.04426234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:28.946536912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:46:30.317227954Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
ts=2026-04-19T19:46:32.855068464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:46:33.043370503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:46:33.043401823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-19T19:46:33.043400694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:46:33.04341186Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:46:34.408898772Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-19T19:46:38.043699225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:38.499623114Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=debug ts=2026-04-19T19:46:42.590223776Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-04-19T19:46:43.043759299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:46.680992436Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-19T19:46:48.044129416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:50.772895556Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=debug ts=2026-04-19T19:46:52.412191264Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:46:53.043852595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:54.863695238Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=info ts=2026-04-19T19:46:57.666442246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:46:57.677799679Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:46:58.043632302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:46:58.953636493Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
ts=2026-04-19T19:47:02.856140046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=debug ts=2026-04-19T19:47:03.023874956Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:47:03.024914306Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:47:03.044287688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:47:03.044306351Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:47:03.044327916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:47:03.044355432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:03.045454895Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=debug ts=2026-04-19T19:47:07.135941517Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-19T19:47:07.819778292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:47:07.826526536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:47:08.044376816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:47:09.175528069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:47228"
level=debug ts=2026-04-19T19:47:11.226951263Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-19T19:47:13.044431531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:15.317869352Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=debug ts=2026-04-19T19:47:15.317899978Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-19T19:47:18.044061205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:18.357054108Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:47:19.409108148Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
ts=2026-04-19T19:47:20.880234092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:38626"
level=warn ts=2026-04-19T19:47:23.043490825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:23.499802652Z caller=checkpoint.go:378 msg="writing series" size="281 kB"
level=debug ts=2026-04-19T19:47:25.981507971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:47:25.983287616Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T19:47:27.590915163Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-19T19:47:28.044072237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:31.680906004Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-04-19T19:47:32.857282855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:47:32.977230207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42700"
level=info ts=2026-04-19T19:47:33.044029535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:47:33.044033211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:33.044049726Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:47:33.044057375Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:47:35.772186945Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-19T19:47:38.043658603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:39.863124568Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=warn ts=2026-04-19T19:47:43.043557206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:43.953978029Z caller=checkpoint.go:378 msg="writing series" size="326 kB"
level=info ts=2026-04-19T19:47:45.33577695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:47:45.344795234Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:47:48.044213197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:48.044545212Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-19T19:47:52.135610483Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-19T19:47:53.043851767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:47:53.10995461Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:47:56.227394399Z caller=checkpoint.go:378 msg="writing series" size="684 kB"
level=debug ts=2026-04-19T19:47:56.227418713Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=10
level=warn ts=2026-04-19T19:47:58.043647691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:48:00.317641567Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-19T19:48:02.859047581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:48:03.024401355Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:48:03.024825201Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:48:03.044054524Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:48:03.044072948Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:48:03.044081991Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:48:03.04413595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:48:03.303036458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:48:03.311514388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:48:04.409129893Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=warn ts=2026-04-19T19:48:08.044032221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:48:08.499792737Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=debug ts=2026-04-19T19:48:12.590422971Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=warn ts=2026-04-19T19:48:13.043432716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:48:15.538392827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:48:16.681361623Z caller=checkpoint.go:378 msg="writing series" size="8.1 kB"
level=warn ts=2026-04-19T19:48:18.043962974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:48:20.772290396Z caller=checkpoint.go:378 msg="writing series" size="416 kB"
ts=2026-04-19T19:48:20.88352089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:39542"
level=warn ts=2026-04-19T19:48:23.043869649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:48:24.863697489Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-19T19:48:25.481137058Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-19T19:48:25.481196566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:48:25.490292884Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T19:48:28.043844972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:48:28.954290291Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
ts=2026-04-19T19:48:32.860165404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:48:33.044256491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:48:33.044259833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:48:33.044280341Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:48:33.044288175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:48:33.045342909Z caller=checkpoint.go:390 msg="flushing series" totalSize="814 kB" series=8
level=info ts=2026-04-19T19:48:33.05252314Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-19T19:48:33.056002661Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.011584815s
level=info ts=2026-04-19T19:48:34.818820073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:48:34.826515193Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:48:38.044079799Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:48:39.056457837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:48:39.063848694Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T19:48:43.043404136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:48:48.043893912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:48:53.044474982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:48:56.338210792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:48:56.345728143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:48:58.044375795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:48:59.092862488Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:48:59.911565308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:49:02.861391492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T19:49:03.02348849Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:49:03.024535349Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:49:03.043581665Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-19T19:49:03.043629468Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:49:03.043634162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:49:03.043649545Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:49:03.043656985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-19T19:49:03.043710815Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-04-19T19:49:03.043892228Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-19T19:49:07.134984063Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-19T19:49:08.044255571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:49:08.315546069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:49:08.325110258Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:49:11.226752703Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T19:49:13.043673489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:49:15.317498515Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-19T19:49:18.044128469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:49:19.408158198Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-19T19:49:23.043486366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:49:23.499174134Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
level=debug ts=2026-04-19T19:49:27.590373939Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-04-19T19:49:27.591433132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:49:27.600244936Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:49:28.044176218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:49:31.680796372Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=debug ts=2026-04-19T19:49:32.525383333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:49:32.862527626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=warn ts=2026-04-19T19:49:33.043801416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:49:33.0438247Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:49:33.04384788Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:49:33.043858159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:49:35.77155931Z caller=checkpoint.go:378 msg="writing series" size="511 B"
level=warn ts=2026-04-19T19:49:38.043614598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:49:39.181696161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:59234"
level=debug ts=2026-04-19T19:49:39.862727783Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-19T19:49:43.043534859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:49:43.953272622Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-19T19:49:48.043741684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:49:48.044309557Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-19T19:49:48.470934017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:49:48.479000991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:49:52.135393167Z caller=checkpoint.go:378 msg="writing series" size="359 kB"
level=debug ts=2026-04-19T19:49:52.135415689Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=13
level=warn ts=2026-04-19T19:49:53.043858172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:49:53.829022099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:49:53.837116731Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:49:56.106583228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:49:56.226155436Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-04-19T19:49:58.044342059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:00.28737172Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:50:00.317623007Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-19T19:50:02.864112945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:50:03.023589815Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:50:03.024659495Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:50:03.044239971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:50:03.044254882Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:50:03.044262747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:50:03.044265922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:04.408393665Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-19T19:50:08.044137811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:08.499402281Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
ts=2026-04-19T19:50:09.182491291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:48272"
level=debug ts=2026-04-19T19:50:12.595463507Z caller=checkpoint.go:378 msg="writing series" size="7.1 MB"
level=debug ts=2026-04-19T19:50:12.595495016Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.4 MB" series=5
level=warn ts=2026-04-19T19:50:13.043561363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:16.680878568Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=warn ts=2026-04-19T19:50:18.044280483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:20.771735209Z caller=checkpoint.go:378 msg="writing series" size="201 kB"
level=warn ts=2026-04-19T19:50:23.044135287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:50:24.288979197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:50:24.300334897Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:50:24.863116432Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
level=warn ts=2026-04-19T19:50:28.04364533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:28.953206202Z caller=checkpoint.go:378 msg="writing series" size="284 kB"
ts=2026-04-19T19:50:32.865756093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:50:32.985412093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:46292"
level=info ts=2026-04-19T19:50:33.043375535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:50:33.043401829Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:50:33.043411406Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:50:33.043407804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:33.044549495Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=debug ts=2026-04-19T19:50:34.026749442Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:50:37.135327767Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=warn ts=2026-04-19T19:50:38.044177318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:50:39.646195403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:50:39.656402163Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:50:41.226070943Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-19T19:50:43.044356955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:45.325097639Z caller=checkpoint.go:378 msg="writing series" size="9.9 MB"
level=debug ts=2026-04-19T19:50:45.325120916Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=8
level=debug ts=2026-04-19T19:50:46.542820467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T19:50:48.044345605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:49.407916981Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-04-19T19:50:51.924131253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:50:51.932893756Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T19:50:53.043650325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:50:53.498797553Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-19T19:50:57.589616628Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-19T19:50:58.043843997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:01.680759689Z caller=checkpoint.go:378 msg="writing series" size="454 kB"
ts=2026-04-19T19:51:02.86723619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:51:03.024233527Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:51:03.024741688Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:51:03.043372215Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:51:03.043388637Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:51:03.04339651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:51:03.043396733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:05.771875928Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-19T19:51:08.043849538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:08.233862383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:51:09.184776377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:52266"
level=debug ts=2026-04-19T19:51:09.864303033Z caller=checkpoint.go:378 msg="writing series" size="3.8 MB"
level=debug ts=2026-04-19T19:51:09.864347368Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.7 MB" series=6
level=warn ts=2026-04-19T19:51:13.04429267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:13.953279411Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-04-19T19:51:14.985163802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:51:14.995954519Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:51:14.995973954Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:51:14.995997953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:51:15.002495868Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:51:18.043622533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:18.044423409Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
ts=2026-04-19T19:51:20.891422952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57262"
level=debug ts=2026-04-19T19:51:22.13534426Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-19T19:51:23.043479742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:26.228878772Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-19T19:51:26.228899783Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=4
level=warn ts=2026-04-19T19:51:28.043837014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:51:28.998174893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:51:29.00890327Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:51:30.316673029Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
ts=2026-04-19T19:51:32.86866442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T19:51:33.044167486Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:51:33.044187094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:51:33.044195347Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:51:33.044191407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:34.408104114Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-19T19:51:36.648902533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:51:36.656488841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:51:38.043745427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:38.499190093Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=debug ts=2026-04-19T19:51:41.202608476Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:51:42.590103495Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=warn ts=2026-04-19T19:51:43.044076058Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:46.404239793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:51:46.405957425Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T19:51:46.680895424Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-19T19:51:48.044271712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:51:50.400287675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:51:50.41003886Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:51:50.772266907Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-04-19T19:51:53.043440779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:54.862404971Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-19T19:51:58.04435212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:51:58.953434769Z caller=checkpoint.go:378 msg="writing series" size="121 kB"
level=info ts=2026-04-19T19:52:00.955776789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:52:00.966046234Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-19T19:52:02.869736843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:52:03.024282718Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:52:03.024345394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:52:03.043610692Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:52:03.043628751Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:52:03.043646371Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:52:03.043653567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:03.044847806Z caller=checkpoint.go:378 msg="writing series" size="351 kB"
level=debug ts=2026-04-19T19:52:07.135718537Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-19T19:52:07.135744235Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-19T19:52:08.043919411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:10.221001396Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:52:11.226677318Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-19T19:52:13.043683834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:15.317248414Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-19T19:52:18.043667702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:19.407843768Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
ts=2026-04-19T19:52:20.893799393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:47398"
level=warn ts=2026-04-19T19:52:23.044218853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:23.4992751Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=debug ts=2026-04-19T19:52:27.59040131Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=warn ts=2026-04-19T19:52:28.043859592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:31.681301408Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-04-19T19:52:32.871187009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:52:33.043393003Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:52:33.043395753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:33.043414599Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:52:33.043423505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:52:33.712346767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-19T19:52:35.757888871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:52:35.76515907Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:52:35.765180635Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:52:35.765205679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:52:35.771238626Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=error ts=2026-04-19T19:52:35.772090467Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-19T19:52:37.384283373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:52:37.392132039Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T19:52:38.043782182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:39.862780862Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
level=debug ts=2026-04-19T19:52:42.548131452Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:52:43.043865864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:52:43.848634598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:52:43.855497104Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:52:43.953818386Z caller=checkpoint.go:378 msg="writing series" size="363 kB"
level=debug ts=2026-04-19T19:52:43.953833653Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=warn ts=2026-04-19T19:52:48.043931852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:48.043972134Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=info ts=2026-04-19T19:52:49.885065583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:52:49.891879213Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:52:52.135671391Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-19T19:52:53.043813912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:52:56.226734589Z caller=checkpoint.go:378 msg="writing series" size="699 kB"
level=warn ts=2026-04-19T19:52:58.044384364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:53:00.317169371Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-19T19:53:02.87258708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:53:03.024547391Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:53:03.025127605Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=warn ts=2026-04-19T19:53:03.044022372Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:53:03.044028496Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:53:03.044047723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:53:03.044055345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:53:04.408728858Z caller=checkpoint.go:378 msg="writing series" size="351 kB"
level=debug ts=2026-04-19T19:53:04.408757672Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-04-19T19:53:08.043839996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:53:08.499366373Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
ts=2026-04-19T19:53:09.189588331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:49028"
level=debug ts=2026-04-19T19:53:12.59012637Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-04-19T19:53:13.04343029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:53:13.391410397Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:53:16.681132847Z caller=checkpoint.go:378 msg="writing series" size="9.0 kB"
level=info ts=2026-04-19T19:53:18.015703628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:53:18.024648774Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:53:18.043439341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:53:20.77181899Z caller=checkpoint.go:378 msg="writing series" size="529 kB"
level=warn ts=2026-04-19T19:53:23.044023004Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:53:24.862370466Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-04-19T19:53:28.043192048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T19:53:28.044288229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-04-19T19:53:28.050022593Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:53:28.072267292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:53:28.953849687Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=info ts=2026-04-19T19:53:30.121055055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:53:30.129371778Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-19T19:53:32.873821184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T19:53:32.993128679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:46218"
level=info ts=2026-04-19T19:53:33.04413933Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T19:53:33.044139857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:53:33.044165202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-04-19T19:53:33.044159302Z caller=checkpoint.go:390 msg="flushing series" totalSize="749 kB" series=6
level=info ts=2026-04-19T19:53:33.044175999Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-19T19:53:33.051447425Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-19T19:53:33.053269467Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00941729s
level=warn ts=2026-04-19T19:53:38.044111566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:53:43.044215691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:53:45.154388534Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:53:48.043945579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:53:53.044340176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:53:58.043736027Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:54:02.875183099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:54:02.994388759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:59632"
level=info ts=2026-04-19T19:54:03.024330686Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:54:03.024359991Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:54:03.043430412Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-19T19:54:03.043465254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:54:03.04348286Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-19T19:54:03.043480995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:54:03.043492781Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-19T19:54:03.043549422Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-04-19T19:54:03.043712259Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-19T19:54:07.134868762Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=debug ts=2026-04-19T19:54:07.511412065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T19:54:08.044442032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:54:11.226469563Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T19:54:13.043698108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:54:13.329573874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:54:13.337832828Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:54:15.316862369Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-04-19T19:54:17.101823926Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:54:18.044459087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:54:19.408401776Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-19T19:54:23.043426854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:54:23.471152617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="83 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:54:23.480194178Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:54:23.49851797Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
level=info ts=2026-04-19T19:54:23.533846464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:54:23.541806695Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:54:27.589919699Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-19T19:54:28.043834498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:54:31.680798538Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
ts=2026-04-19T19:54:32.876149577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T19:54:33.043384542Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:54:33.043408442Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:54:33.043421656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:54:33.043424106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:54:35.771337701Z caller=checkpoint.go:378 msg="writing series" size="511 B"
level=warn ts=2026-04-19T19:54:38.043821807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:54:39.862708692Z caller=checkpoint.go:378 msg="writing series" size="215 kB"
level=debug ts=2026-04-19T19:54:42.894042156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:54:43.043588716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:54:43.952905353Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=debug ts=2026-04-19T19:54:43.952929289Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-04-19T19:54:48.043750716Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-19T19:54:48.043761094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:54:52.135179621Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=warn ts=2026-04-19T19:54:53.044130854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:54:54.192730172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:54:54.200537888Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:54:56.226693262Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-19T19:54:58.044323703Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:00.316824006Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-19T19:55:02.876951842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=debug ts=2026-04-19T19:55:03.02369788Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:55:03.024764393Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:55:03.04377092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:55:03.043788495Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:55:03.043796808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:55:03.043822572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:04.408067173Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-04-19T19:55:04.650821775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-19T19:55:06.709657045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:55:06.718178838Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:55:06.718200059Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:55:06.718223652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:55:06.724795618Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T19:55:08.043486745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:08.498363998Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
ts=2026-04-19T19:55:09.196247627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:36486"
level=debug ts=2026-04-19T19:55:09.796277573Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:55:12.596273249Z caller=checkpoint.go:378 msg="writing series" size="7.4 MB"
level=debug ts=2026-04-19T19:55:12.596303805Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.9 MB" series=7
level=info ts=2026-04-19T19:55:12.914905436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="83 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:55:12.922605359Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:55:13.04375956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:16.68087642Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=warn ts=2026-04-19T19:55:18.044050827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:20.771953628Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-19T19:55:23.043901777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:55:24.23392196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:55:24.245236346Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:55:24.862886238Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-19T19:55:28.043556824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:28.954057381Z caller=checkpoint.go:378 msg="writing series" size="447 kB"
ts=2026-04-19T19:55:32.878105982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T19:55:33.044285174Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:55:33.044324637Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:55:33.044333159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:55:33.04437104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:33.044424475Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=debug ts=2026-04-19T19:55:33.044436397Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-04-19T19:55:36.508778126Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-19T19:55:36.66837136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:55:36.677415882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:55:37.134786415Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-19T19:55:38.044244171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:41.2256668Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-19T19:55:43.043528598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:45.323572562Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-19T19:55:45.323596253Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=3
level=warn ts=2026-04-19T19:55:48.043685438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:49.408442415Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-04-19T19:55:49.685285342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:55:49.696173278Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-aoqw\"}"
ts=2026-04-19T19:55:50.903390065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:44148"
level=debug ts=2026-04-19T19:55:52.498069351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:55:52.499973311Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:55:53.044182671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:55:53.499237208Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-19T19:55:57.590091555Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=warn ts=2026-04-19T19:55:58.043559371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:55:58.77730978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="83 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:55:58.785822866Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:56:01.680520092Z caller=checkpoint.go:378 msg="writing series" size="501 kB"
ts=2026-04-19T19:56:02.879201554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:56:03.000534862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:51498"
level=debug ts=2026-04-19T19:56:03.023924383Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:56:03.025000436Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:56:03.04434072Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:56:03.044365299Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:56:03.044376574Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:56:03.044394503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:05.772025082Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-19T19:56:07.597355613Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:56:08.043609895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:56:09.198692346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58960"
level=debug ts=2026-04-19T19:56:09.864973994Z caller=checkpoint.go:378 msg="writing series" size="6.1 MB"
level=debug ts=2026-04-19T19:56:09.864996756Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.1 MB" series=6
level=warn ts=2026-04-19T19:56:13.044437423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:13.952929204Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=warn ts=2026-04-19T19:56:18.043967225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:18.044243482Z caller=checkpoint.go:378 msg="writing series" size="614 kB"
level=info ts=2026-04-19T19:56:18.481847172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:56:18.489424753Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:56:22.134970053Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-19T19:56:23.043400669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:26.228216949Z caller=checkpoint.go:378 msg="writing series" size="1.9 MB"
level=debug ts=2026-04-19T19:56:26.228237169Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=4
level=warn ts=2026-04-19T19:56:28.043468686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:30.316749233Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-19T19:56:32.880775685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=warn ts=2026-04-19T19:56:33.043588357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:56:33.043604709Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:56:33.043621304Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:56:33.043629107Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:56:34.251067375Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:56:34.408386127Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=debug ts=2026-04-19T19:56:35.818244371Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T19:56:38.04355038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:38.498358258Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=info ts=2026-04-19T19:56:40.288049514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:56:40.295788445Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:56:42.590007563Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-04-19T19:56:43.043838494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:56:46.221287655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="83 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:56:46.22939587Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:56:46.680455655Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-19T19:56:48.043719593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:50.771711355Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=warn ts=2026-04-19T19:56:53.043510192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:54.86289034Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-19T19:56:58.04437538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:56:58.953331151Z caller=checkpoint.go:378 msg="writing series" size="177 kB"
level=debug ts=2026-04-19T19:56:59.855450095Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-19T19:57:02.881829662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:57:03.002778567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42322"
level=info ts=2026-04-19T19:57:03.024580174Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T19:57:03.025033024Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:57:03.043370313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:57:03.043386639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:57:03.043394045Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:57:03.043395787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:03.044532321Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
level=debug ts=2026-04-19T19:57:03.044543431Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-04-19T19:57:07.135547582Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-19T19:57:08.043563549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:11.225635621Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=warn ts=2026-04-19T19:57:13.044378615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:15.316779689Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-04-19T19:57:16.857898646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:57:16.869230793Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:57:18.04372213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:19.408184602Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-19T19:57:23.043437661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:23.499095865Z caller=checkpoint.go:378 msg="writing series" size="176 kB"
level=debug ts=2026-04-19T19:57:23.971405926Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:57:25.791871251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:57:27.589623991Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=warn ts=2026-04-19T19:57:28.043984574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:31.680718557Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
ts=2026-04-19T19:57:32.883410091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:57:33.004334949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:44698"
level=info ts=2026-04-19T19:57:33.043414427Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:57:33.043447921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:57:33.043461419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:57:33.043462045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:35.77123961Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-19T19:57:37.318973076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:57:37.32768163Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:57:37.327700396Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:57:37.327725007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:57:37.333945035Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-19T19:57:37.675582938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:57:37.683001497Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T19:57:38.044266748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:57:39.202183142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:39192"
level=debug ts=2026-04-19T19:57:39.862725534Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=warn ts=2026-04-19T19:57:43.044030644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:43.953962992Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=debug ts=2026-04-19T19:57:43.953987449Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-04-19T19:57:48.043856553Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-19T19:57:48.043860828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:57:48.659870123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:57:52.134865818Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-19T19:57:53.044194446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:57:53.341030115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:57:53.348653838Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:57:56.226686939Z caller=checkpoint.go:378 msg="writing series" size="951 kB"
level=debug ts=2026-04-19T19:57:56.22670962Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-19T19:57:58.044055861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:58:00.316881679Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-19T19:58:01.257680442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:58:01.265122992Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-19T19:58:02.88513191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T19:58:03.023399964Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:58:03.024456165Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:58:03.044053598Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:58:03.044069536Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:58:03.044076952Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:58:03.044078748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:58:04.407827851Z caller=checkpoint.go:378 msg="writing series" size="463 kB"
level=warn ts=2026-04-19T19:58:08.043853057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:58:08.499188699Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
ts=2026-04-19T19:58:09.203791769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:37152"
level=debug ts=2026-04-19T19:58:12.589475049Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=warn ts=2026-04-19T19:58:13.043465537Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:58:16.681069357Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-19T19:58:18.044370582Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:58:19.308457108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:58:19.318823629Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T19:58:20.771763817Z caller=checkpoint.go:378 msg="writing series" size="533 kB"
level=debug ts=2026-04-19T19:58:20.771789676Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-04-19T19:58:21.060766582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:58:23.043782134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:58:23.158095048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:58:23.165988435Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:58:24.863329379Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-19T19:58:27.583917197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T19:58:28.044090632Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:58:28.95373444Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-19T19:58:32.886748608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
ts=2026-04-19T19:58:33.007525752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:39418"
level=info ts=2026-04-19T19:58:33.043663629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:58:33.043690682Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:58:33.043704226Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:58:33.04370666Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 kB" series=2
level=warn ts=2026-04-19T19:58:33.043733144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:58:33.136601469Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-19T19:58:33.137429122Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.093752307s
level=warn ts=2026-04-19T19:58:38.044135779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:58:43.04346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:58:43.917099201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:58:43.926191103Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:58:43.926216898Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T19:58:43.926256328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:58:43.933700363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T19:58:46.044911879Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:58:48.044233728Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:58:53.044223215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T19:58:58.044242918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:59:01.802207487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:59:01.810975936Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-19T19:59:02.00557576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:59:02.012453838Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-19T19:59:02.88842691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
ts=2026-04-19T19:59:03.00948559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:57790"
level=debug ts=2026-04-19T19:59:03.023767316Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T19:59:03.024781255Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T19:59:03.04394583Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-19T19:59:03.043954074Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-19T19:59:03.043964706Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:59:03.043976368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T19:59:03.043991884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:59:03.044132909Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-04-19T19:59:03.044294653Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-19T19:59:05.71245458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:59:05.720296944Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T19:59:07.135458683Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T19:59:08.044168622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T19:59:09.207417051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:51674"
level=debug ts=2026-04-19T19:59:09.904882625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T19:59:11.226869921Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-19T19:59:12.93902124Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T19:59:13.044139937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:59:15.317966837Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-19T19:59:18.044199284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:59:19.408632322Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-19T19:59:23.04429524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:59:23.500089544Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
level=debug ts=2026-04-19T19:59:27.590379181Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-19T19:59:28.044360203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:59:31.681126405Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
ts=2026-04-19T19:59:32.889827036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=warn ts=2026-04-19T19:59:33.043584347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T19:59:33.043580447Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T19:59:33.043609447Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T19:59:33.043620135Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T19:59:35.771675714Z caller=checkpoint.go:378 msg="writing series" size="511 B"
level=info ts=2026-04-19T19:59:37.575685547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T19:59:37.584950857Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T19:59:38.04413415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:59:39.863518879Z caller=checkpoint.go:378 msg="writing series" size="328 kB"
level=debug ts=2026-04-19T19:59:39.863540134Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-19T19:59:43.043436034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:59:43.145882589Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T19:59:43.954394828Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-19T19:59:48.043822936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:59:48.044589324Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
ts=2026-04-19T19:59:50.913448515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41512"
level=debug ts=2026-04-19T19:59:52.13624324Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=info ts=2026-04-19T19:59:52.922350497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T19:59:52.929250309Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T19:59:53.044256948Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T19:59:53.429715605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:59:53.431497092Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T19:59:56.227110553Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=info ts=2026-04-19T19:59:57.402062039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T19:59:57.408675284Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T19:59:58.043533737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:00.317200258Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-19T20:00:02.89127302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T20:00:03.012670054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:37786"
level=debug ts=2026-04-19T20:00:03.02414379Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:00:03.025205313Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T20:00:03.025433419Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-19T20:00:03.025501927Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974031 n=1
level=debug ts=2026-04-19T20:00:03.025563757Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974031 n=1
level=info ts=2026-04-19T20:00:03.044047906Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:00:03.044073468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:00:03.044081964Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:00:03.044102761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:04.408225113Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-19T20:00:08.043950097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:08.4994563Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
ts=2026-04-19T20:00:09.209198122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:46712"
level=debug ts=2026-04-19T20:00:12.596104637Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-04-19T20:00:12.596129831Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.0 MB" series=8
level=warn ts=2026-04-19T20:00:13.044091495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:16.683588869Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=debug ts=2026-04-19T20:00:17.351722619Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T20:00:18.044026892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:20.772516712Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=warn ts=2026-04-19T20:00:23.044203119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:24.862835338Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=info ts=2026-04-19T20:00:27.345706905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:00:27.357530014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T20:00:28.044098778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:28.953744367Z caller=checkpoint.go:378 msg="writing series" size="474 kB"
level=debug ts=2026-04-19T20:00:28.953771572Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-04-19T20:00:32.892894754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T20:00:33.044295316Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:00:33.044342212Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:00:33.044350959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:00:33.04434829Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:33.044485566Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=debug ts=2026-04-19T20:00:37.136127395Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=warn ts=2026-04-19T20:00:38.044433804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T20:00:39.21178371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:43846"
level=info ts=2026-04-19T20:00:39.681073066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:00:39.692737924Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T20:00:41.226687285Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-19T20:00:43.043919366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:45.333854938Z caller=checkpoint.go:378 msg="writing series" size="16 MB"
level=debug ts=2026-04-19T20:00:45.333883362Z caller=checkpoint.go:390 msg="flushing series" totalSize="16 MB" series=4
level=debug ts=2026-04-19T20:00:46.042922581Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:00:47.701177991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T20:00:48.044386824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:00:48.929474844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:00:48.937355226Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T20:00:49.408579194Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=warn ts=2026-04-19T20:00:53.043408756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:00:53.499551825Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-19T20:00:57.590654001Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=warn ts=2026-04-19T20:00:58.043606065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:01.68182115Z caller=checkpoint.go:378 msg="writing series" size="583 kB"
level=debug ts=2026-04-19T20:01:01.6818497Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-04-19T20:01:02.894207107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T20:01:03.024236206Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T20:01:03.024628983Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:01:03.044286956Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:01:03.044304592Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:01:03.044327941Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:01:03.044335884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:05.772491089Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-19T20:01:08.043810881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:09.870601811Z caller=checkpoint.go:378 msg="writing series" size="9.3 MB"
level=debug ts=2026-04-19T20:01:09.870626847Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.3 MB" series=2
level=warn ts=2026-04-19T20:01:13.04413841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:13.95428849Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=warn ts=2026-04-19T20:01:18.043477084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:18.045066202Z caller=checkpoint.go:378 msg="writing series" size="602 kB"
level=info ts=2026-04-19T20:01:20.207011447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:01:20.216769044Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:01:20.216790122Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T20:01:20.216813756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:01:20.223864696Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T20:01:20.646150836Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:01:22.135270377Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-19T20:01:23.043652861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:01:23.655995599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:01:23.663578985Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T20:01:26.230328675Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-19T20:01:26.230351694Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.1 MB" series=4
level=warn ts=2026-04-19T20:01:28.043603763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:30.317298583Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=info ts=2026-04-19T20:01:30.606811872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:01:30.617122698Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
ts=2026-04-19T20:01:32.895361732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
ts=2026-04-19T20:01:33.016365262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:38678"
level=info ts=2026-04-19T20:01:33.044135811Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:01:33.044166263Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:01:33.044177343Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:01:33.044174232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:34.408652846Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-04-19T20:01:37.492243727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T20:01:38.043684379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:38.49955122Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
ts=2026-04-19T20:01:39.214866665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:40416"
level=debug ts=2026-04-19T20:01:42.59025672Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-19T20:01:43.043618339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:45.452517167Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:01:46.680915569Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-19T20:01:47.26779435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:01:47.27600712Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T20:01:48.044368282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:50.771981708Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=warn ts=2026-04-19T20:01:53.043666924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:54.862925794Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=warn ts=2026-04-19T20:01:58.044245679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:01:58.954448668Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
ts=2026-04-19T20:02:02.897011441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T20:02:03.024208569Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T20:02:03.024564622Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:02:03.043953358Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:02:03.043974643Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:02:03.043982298Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:02:03.044006798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:03.045345516Z caller=checkpoint.go:378 msg="writing series" size="384 kB"
level=debug ts=2026-04-19T20:02:03.045359293Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-04-19T20:02:07.13582302Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-19T20:02:07.149918189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:02:07.161240668Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T20:02:08.043865825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:02:09.103246128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:02:09.110951618Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T20:02:11.2270611Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=warn ts=2026-04-19T20:02:13.044291297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:15.317594366Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=debug ts=2026-04-19T20:02:15.561340428Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T20:02:18.044434331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:19.408216112Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-19T20:02:23.043973005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:23.499036546Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=debug ts=2026-04-19T20:02:27.119585741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T20:02:27.590672261Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=warn ts=2026-04-19T20:02:28.04381922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:02:28.223703502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:02:28.230442039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T20:02:31.680934576Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
ts=2026-04-19T20:02:32.898294592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T20:02:33.043998371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:02:33.044023872Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:02:33.044033696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:02:33.044026417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:35.772231427Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-19T20:02:38.044424918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:39.86322604Z caller=checkpoint.go:378 msg="writing series" size="305 kB"
level=debug ts=2026-04-19T20:02:39.863247356Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-19T20:02:43.04386169Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:43.954243304Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=debug ts=2026-04-19T20:02:44.357580362Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T20:02:48.044077515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:48.044638749Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-19T20:02:52.135608674Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-19T20:02:53.04356594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:02:56.226948867Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-19T20:02:56.226971884Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-04-19T20:02:58.044403635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:02:59.915844781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:02:59.924643383Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T20:03:00.317890274Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-19T20:03:00.720193471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:03:00.726965438Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-19T20:03:02.899970584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T20:03:03.023757256Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:03:03.024822807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T20:03:03.044348746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:03:03.044364298Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:03:03.044372336Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:03:03.044372021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:04.093476798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T20:03:04.408480245Z caller=checkpoint.go:378 msg="writing series" size="352 kB"
level=warn ts=2026-04-19T20:03:08.04420426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:08.49989821Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=debug ts=2026-04-19T20:03:12.589948646Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=warn ts=2026-04-19T20:03:13.044064775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:16.681473094Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-19T20:03:18.044250858Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:18.670905971Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-19T20:03:20.098758148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:03:20.106589539Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T20:03:20.772685318Z caller=checkpoint.go:378 msg="writing series" size="619 kB"
level=debug ts=2026-04-19T20:03:20.772708079Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-04-19T20:03:20.923861644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53876"
level=warn ts=2026-04-19T20:03:23.044464211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:24.862983616Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-19T20:03:28.043859813Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:28.953625625Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
ts=2026-04-19T20:03:32.901629544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
ts=2026-04-19T20:03:33.021640599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:57420"
level=info ts=2026-04-19T20:03:33.044021407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:03:33.044039703Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:03:33.044047007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:03:33.044049404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:33.045119418Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 kB" series=2
level=info ts=2026-04-19T20:03:33.048460834Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-19T20:03:33.049426867Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005171315s
level=warn ts=2026-04-19T20:03:38.043573661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:03:42.606333354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:03:42.614520222Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T20:03:43.043650106Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:43.810391865Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T20:03:48.044425986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T20:03:53.044093252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:03:55.580204453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:03:55.582118527Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T20:03:57.721115071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:03:57.730076748Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T20:03:58.044476832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T20:04:02.902808505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T20:04:03.023733288Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:04:03.024775866Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T20:04:03.043995165Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:04:03.044012016Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:04:03.04400385Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-19T20:04:03.044019329Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:04:03.044039403Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:04:03.044102518Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-04-19T20:04:03.044298705Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-19T20:04:07.136050951Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-19T20:04:08.044116685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T20:04:09.221800117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:58580"
level=debug ts=2026-04-19T20:04:11.227050689Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-19T20:04:11.753399482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T20:04:13.043613525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:04:15.317947199Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=info ts=2026-04-19T20:04:16.103078236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:04:16.113015208Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:04:16.113036154Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T20:04:16.113064036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:04:16.120208073Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T20:04:18.043422679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:04:19.408120794Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-19T20:04:23.043824248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:04:23.50012196Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
level=debug ts=2026-04-19T20:04:27.590392276Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-19T20:04:28.043759867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:04:29.226278324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:04:29.236817388Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
level=info ts=2026-04-19T20:04:30.012627787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:04:30.019916762Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T20:04:31.681287176Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
ts=2026-04-19T20:04:32.904692575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T20:04:33.023159624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42392"
level=info ts=2026-04-19T20:04:33.044359096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T20:04:33.044362071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:04:33.044380017Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:04:33.044392859Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T20:04:35.772367256Z caller=checkpoint.go:378 msg="writing series" size="643 B"
level=warn ts=2026-04-19T20:04:38.043798033Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:04:39.140435902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T20:04:39.863653561Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=warn ts=2026-04-19T20:04:43.043914219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:04:43.865123924Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:04:43.954392702Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=debug ts=2026-04-19T20:04:43.95441322Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=debug ts=2026-04-19T20:04:48.044335559Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-19T20:04:48.04436369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:04:52.135455734Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
level=info ts=2026-04-19T20:04:52.879837899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:04:52.889442564Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T20:04:53.044352538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:04:56.227435814Z caller=checkpoint.go:378 msg="writing series" size="295 kB"
level=warn ts=2026-04-19T20:04:58.044255638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:00.317171675Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-19T20:05:02.905842015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=debug ts=2026-04-19T20:05:03.023731287Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:05:03.024784101Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T20:05:03.044007788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:05:03.044024791Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:05:03.04403277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:05:03.04403453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:05:04.392301143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:05:04.400769063Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T20:05:04.408864046Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-19T20:05:08.043999652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:08.498902488Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=debug ts=2026-04-19T20:05:12.596429315Z caller=checkpoint.go:378 msg="writing series" size="8.5 MB"
level=debug ts=2026-04-19T20:05:12.596451487Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.2 MB" series=7
level=warn ts=2026-04-19T20:05:13.044139365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:05:13.740610284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:05:13.748135648Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T20:05:14.194397963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:05:16.681162774Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=warn ts=2026-04-19T20:05:18.043614031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:20.772675571Z caller=checkpoint.go:378 msg="writing series" size="203 kB"
level=warn ts=2026-04-19T20:05:23.043710204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:23.068445171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T20:05:24.862928563Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-19T20:05:28.043778086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:28.953970628Z caller=checkpoint.go:378 msg="writing series" size="409 kB"
ts=2026-04-19T20:05:32.907225359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=info ts=2026-04-19T20:05:33.044146751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:05:33.044167757Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:05:33.044177917Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:05:33.044170931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:33.045439917Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=debug ts=2026-04-19T20:05:33.045449772Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-04-19T20:05:37.13610781Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-19T20:05:38.043890892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:41.226848521Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-19T20:05:43.043735126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:45.336341625Z caller=checkpoint.go:378 msg="writing series" size="18 MB"
level=debug ts=2026-04-19T20:05:45.336364877Z caller=checkpoint.go:390 msg="flushing series" totalSize="18 MB" series=3
level=info ts=2026-04-19T20:05:47.283283734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:05:47.29155126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T20:05:48.044013074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:48.889602729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:05:49.408865544Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=info ts=2026-04-19T20:05:51.288122619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:05:51.295228329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:05:51.295250738Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T20:05:51.295277619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:05:51.302290994Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T20:05:53.043763763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:05:53.499761742Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-19T20:05:57.590666467Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-19T20:05:58.043694725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:01.681791221Z caller=checkpoint.go:378 msg="writing series" size="672 kB"
level=debug ts=2026-04-19T20:06:01.681815664Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-04-19T20:06:02.908458043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=debug ts=2026-04-19T20:06:03.024198296Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:06:03.024231125Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-19T20:06:03.026951774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:43338"
level=info ts=2026-04-19T20:06:03.043705875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:06:03.043730384Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:06:03.043741451Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:06:03.043766092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:05.771917166Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-19T20:06:08.043848201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:06:08.728258422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:06:08.736183779Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-aoqw\"}"
ts=2026-04-19T20:06:09.227380245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:59590"
level=info ts=2026-04-19T20:06:09.75312179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:06:09.760161533Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T20:06:09.874883358Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-19T20:06:09.874909307Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=2
level=warn ts=2026-04-19T20:06:13.044366166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:13.954158438Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=debug ts=2026-04-19T20:06:17.83571Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:06:17.971517462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T20:06:18.044217386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:18.044839197Z caller=checkpoint.go:378 msg="writing series" size="888 kB"
level=debug ts=2026-04-19T20:06:18.044851187Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
ts=2026-04-19T20:06:20.931660493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:49396"
level=debug ts=2026-04-19T20:06:22.135752976Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-19T20:06:23.044095979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:26.227518307Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-19T20:06:26.22754149Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=2
level=warn ts=2026-04-19T20:06:28.043618938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:30.318388511Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=info ts=2026-04-19T20:06:30.451763229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="157 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:06:30.463554089Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-19T20:06:31.658433248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:06:31.666438581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-19T20:06:32.909972214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T20:06:33.04380245Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T20:06:33.04380045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:33.043823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:06:33.043831016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T20:06:34.408349203Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=warn ts=2026-04-19T20:06:38.04390434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:38.499099871Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=debug ts=2026-04-19T20:06:42.590717424Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=warn ts=2026-04-19T20:06:43.043860575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:44.520093598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:06:46.681055724Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=warn ts=2026-04-19T20:06:48.04443608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:50.77247648Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
ts=2026-04-19T20:06:50.933191282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41060"
level=warn ts=2026-04-19T20:06:53.044346657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:54.112055147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T20:06:54.863574296Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=info ts=2026-04-19T20:06:57.900968874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:06:57.909928295Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T20:06:58.044098729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:06:58.954435808Z caller=checkpoint.go:378 msg="writing series" size="281 kB"
ts=2026-04-19T20:07:02.911292538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T20:07:03.023944916Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:07:03.024989741Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-19T20:07:03.043570862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:07:03.043588052Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:07:03.043596253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:07:03.043601502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:03.044888346Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=debug ts=2026-04-19T20:07:03.044898326Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-04-19T20:07:07.136303469Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-19T20:07:08.044425044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:11.22664192Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=warn ts=2026-04-19T20:07:13.043454311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:07:13.379286766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="157 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:07:13.39009539Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-19T20:07:15.294824664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:07:15.301525466Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T20:07:15.317760566Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=warn ts=2026-04-19T20:07:18.043844908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:18.385026046Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:07:19.40856269Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
ts=2026-04-19T20:07:20.934616725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:50362"
level=warn ts=2026-04-19T20:07:23.043834568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:23.499426785Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=debug ts=2026-04-19T20:07:27.590337802Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=warn ts=2026-04-19T20:07:28.044151991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:31.681630601Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
ts=2026-04-19T20:07:32.912175376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T20:07:33.043588655Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T20:07:33.043598718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:33.043613537Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:07:33.043628565Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T20:07:35.771918098Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-19T20:07:38.043802384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:39.863062267Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=debug ts=2026-04-19T20:07:39.863086908Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-04-19T20:07:42.770039053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T20:07:43.043454415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:43.953978588Z caller=checkpoint.go:378 msg="writing series" size="456 kB"
level=info ts=2026-04-19T20:07:44.72014626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:07:44.731342453Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:07:44.731359203Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T20:07:44.731384125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:07:44.738075061Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T20:07:48.044003993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:48.044855842Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-19T20:07:52.135740634Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=debug ts=2026-04-19T20:07:52.332101995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:07:52.333779757Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T20:07:53.0438217Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:07:56.226937507Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-19T20:07:56.226960906Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-04-19T20:07:58.043936484Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:07:58.606042088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:07:58.613822501Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-19T20:08:00.155281228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:08:00.16230527Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T20:08:00.317797003Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-19T20:08:02.913254171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T20:08:03.02375466Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:08:03.024800387Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-19T20:08:03.031201274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:49680"
level=info ts=2026-04-19T20:08:03.043674743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:08:03.04369199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:08:03.043700316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:08:03.043717657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:08:04.409075875Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
level=warn ts=2026-04-19T20:08:08.043741672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:08:08.499188852Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=info ts=2026-04-19T20:08:08.572427444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="157 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:08:08.58109257Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T20:08:12.589996973Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=debug ts=2026-04-19T20:08:12.72157332Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T20:08:13.043806826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:08:16.681603979Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-19T20:08:18.044164878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:08:20.772764002Z caller=checkpoint.go:378 msg="writing series" size="702 kB"
level=debug ts=2026-04-19T20:08:20.772786625Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-04-19T20:08:23.044280189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:08:24.862814942Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-19T20:08:28.043427005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:08:28.954301294Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
ts=2026-04-19T20:08:32.914828875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
ts=2026-04-19T20:08:33.03264702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:33388"
level=info ts=2026-04-19T20:08:33.043403986Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-19T20:08:33.043408657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:08:33.043422254Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:08:33.043431002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-19T20:08:33.044499272Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 kB" series=2
level=info ts=2026-04-19T20:08:33.047964555Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-19T20:08:33.049270638Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005013868s
level=info ts=2026-04-19T20:08:36.546715219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:08:36.556457909Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T20:08:38.04434688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:08:40.48859152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T20:08:42.123361599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-19T20:08:43.04414134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T20:08:48.043559768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-19T20:08:53.043595289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:08:53.132882891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:08:53.141108039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-19T20:08:58.044449827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T20:09:02.916278149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=info ts=2026-04-19T20:09:03.024400085Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-19T20:09:03.024401653Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:09:03.043676316Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-19T20:09:03.043688692Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:09:03.043710397Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-19T20:09:03.043714479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:09:03.043721149Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-19T20:09:03.04380239Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=debug ts=2026-04-19T20:09:03.043976643Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-19T20:09:06.629430607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="157 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:09:06.638408835Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T20:09:07.135824925Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=warn ts=2026-04-19T20:09:08.043841909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T20:09:09.235220691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:45670"
level=debug ts=2026-04-19T20:09:11.226831012Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-19T20:09:13.044365685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:09:13.156735104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:09:15.317654485Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=warn ts=2026-04-19T20:09:18.044087265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:09:19.408230258Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-19T20:09:23.044202581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:09:23.499665849Z caller=checkpoint.go:378 msg="writing series" size="517 kB"
level=debug ts=2026-04-19T20:09:24.598676144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-19T20:09:27.590424854Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-19T20:09:28.043913188Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:09:31.680585092Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
ts=2026-04-19T20:09:32.917248427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-vjt5t-93411d0d 10.128.0.65:7946"
level=warn ts=2026-04-19T20:09:33.044047235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:09:33.044048112Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:09:33.044068123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:09:33.04407536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-19T20:09:34.746281198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:09:34.754970125Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T20:09:35.771603647Z caller=checkpoint.go:378 msg="writing series" size="643 B"
level=info ts=2026-04-19T20:09:36.155724798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:09:36.163219148Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-aoqw\"}"
level=warn ts=2026-04-19T20:09:38.043706062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:09:38.517167117Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:09:39.863311285Z caller=checkpoint.go:378 msg="writing series" size="318 kB"
level=debug ts=2026-04-19T20:09:39.863354885Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-19T20:09:43.043632155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:09:43.954101497Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=debug ts=2026-04-19T20:09:48.044416246Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-19T20:09:48.044438971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-19T20:09:50.941140258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56920"
level=debug ts=2026-04-19T20:09:52.135196644Z caller=checkpoint.go:378 msg="writing series" size="328 kB"
level=warn ts=2026-04-19T20:09:53.044171055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:09:56.226643502Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-04-19T20:09:56.489293657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="157 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:09:56.500535377Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-19T20:09:58.043858323Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:00.316824205Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-19T20:10:02.919036773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-4pqqh-f83ec366 10.128.0.67:7946"
level=debug ts=2026-04-19T20:10:03.023437194Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.86:11211 count=1
level=info ts=2026-04-19T20:10:03.024483127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-19T20:10:03.035218786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:48266"
level=info ts=2026-04-19T20:10:03.043976212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:10:03.043993896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:10:03.044002863Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:10:03.044002311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:04.408387617Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=debug ts=2026-04-19T20:10:04.995661412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T20:10:08.043562489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:08.498565192Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
ts=2026-04-19T20:10:09.238173487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.65:44814"
level=debug ts=2026-04-19T20:10:10.951660443Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:10:12.59803572Z caller=checkpoint.go:378 msg="writing series" size="9.1 MB"
level=debug ts=2026-04-19T20:10:12.598059988Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.7 MB" series=8
level=warn ts=2026-04-19T20:10:13.043729547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:16.680548502Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=warn ts=2026-04-19T20:10:18.043861784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:10:18.926545476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-19T20:10:18.935627713Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19da72e9f59%3A19da72f7092%3A4db5e1d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-19T20:10:20.771677501Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=warn ts=2026-04-19T20:10:23.044238337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:10:24.42496414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc97ed22003ff6cf immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-aoqw\"}"
level=error ts=2026-04-19T20:10:24.432181303Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc97ed22003ff6cf/19da73cc134%3A19da7491890%3A60312dad?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-aoqw\"}"
level=debug ts=2026-04-19T20:10:24.862819856Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-04-19T20:10:28.044227533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:28.953539865Z caller=checkpoint.go:378 msg="writing series" size="552 kB"
level=debug ts=2026-04-19T20:10:28.953562661Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-04-19T20:10:32.920495098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-8b34eb38 10.128.0.68:7946"
level=info ts=2026-04-19T20:10:33.044121184Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-19T20:10:33.044140475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-19T20:10:33.044149443Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-19T20:10:33.044180462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:33.044306169Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
level=debug ts=2026-04-19T20:10:37.135572517Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=warn ts=2026-04-19T20:10:38.044435602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:38.686883299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-19T20:10:41.226058837Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-19T20:10:43.043663929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:45.328231545Z caller=checkpoint.go:378 msg="writing series" size="20 MB"
level=debug ts=2026-04-19T20:10:45.328256261Z caller=checkpoint.go:390 msg="flushing series" totalSize="21 MB" series=4
level=debug ts=2026-04-19T20:10:45.525705673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-19T20:10:48.043893706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-19T20:10:48.575739848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="157 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:10:48.582785778Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:10:48.582807556Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-19T20:10:48.582835822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="157 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-19T20:10:48.589383005Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19da72e9eb3%3A19da73f227e%3A2a520fde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-19T20:10:49.408731155Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
ts=2026-04-19T20:10:50.943920069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:48858"
level=warn ts=2026-04-19T20:10:53.043579635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-19T20:10:53.499607061Z caller=checkpoint.go:378 msg="writing series" size="224 kB"