level=info ts=2026-04-17T09:14:16.599693315Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-17T09:14:16.599722573Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-17T09:14:16.600163274Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-17T09:14:16.601149913Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-17T09:14:16.6016553Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-17T09:14:16.601745376Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.90 inf=eth0
level=info ts=2026-04-17T09:14:16.604480546Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-c2a8d9c9
ts=2026-04-17T09:14:16.604548785Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-17T09:14:16.604562226Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.90 advertisePort=7946
level=debug ts=2026-04-17T09:14:16.604580075Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.90 advertisePort=7946
level=info ts=2026-04-17T09:14:16.604711037Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-17T09:14:16.607933983Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-17T09:14:16.607965858Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-04-17T09:14:16.609285449Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-17T09:14:16.60969459Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-17T09:14:16.609710874Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-17T09:14:16.609708935Z
level=debug ts=2026-04-17T09:14:16.609728149Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-17T09:14:16.609747465Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-17T09:14:16.609772299Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-17T09:14:16.609780026Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-17T09:14:16.609778412Z
level=debug ts=2026-04-17T09:14:16.609787021Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-17T09:14:16.609989956Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-17T09:14:16.612143339Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-17T09:14:16.612166888Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=info ts=2026-04-17T09:14:16.612167402Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:14:16.612194994Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.48458ms
level=error ts=2026-04-17T09:14:16.612208632Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-17T09:14:16.612235842Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-04-17T09:14:16.612272614Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.90 inf=eth0
ts=2026-04-17T09:14:16.615379889Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:16.615394715Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T09:14:16.627389853Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-17T09:14:16.627411616Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=info ts=2026-04-17T09:14:16.627421604Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-17T09:14:16.627439509Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-04-17T09:14:16.627452219Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-17T09:14:16.627461545Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-17T09:14:16.627475203Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=info ts=2026-04-17T09:14:16.627483812Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-17T09:14:16.627492922Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-17T09:14:16.627496928Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-17T09:14:16.627674495Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-17T09:14:16.627683042Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-04-17T09:14:16.627679665Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-04-17T09:14:16.627686709Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-17T09:14:16.627687005Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-17T09:14:16.62769295Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-04-17T09:14:16.627692338Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-04-17T09:14:16.627698416Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-04-17T09:14:16.627750285Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-17T09:14:16.627786084Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-17T09:14:16.627838051Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-17T09:14:16.627857635Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-17T09:14:16.627864398Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-04-17T09:14:16.627869395Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-04-17T09:14:16.62790322Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-17T09:14:16.627915911Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-17T09:14:16.627943166Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-17T09:14:16.627995402Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=80.595µs errors=false
level=info ts=2026-04-17T09:14:16.628030042Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-17T09:14:16.628220319Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=305.635µs errors=false
level=info ts=2026-04-17T09:14:16.628235145Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-17T09:14:16.628247331Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=332.702µs
level=info ts=2026-04-17T09:14:16.628308755Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-17T09:14:16.628330069Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-04-17T09:14:16.628330578Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-17T09:14:16.6283644Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=14.565391936s
level=debug ts=2026-04-17T09:14:16.628426056Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-17T09:14:16.628434822Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-04-17T09:14:16.628470182Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=debug ts=2026-04-17T09:14:16.628484464Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-17T09:14:16.628483837Z caller=loki.go:599 msg="Loki started" startup_time=111.880366ms
ts=2026-04-17T09:14:17.87991784Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:17.879939306Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-17T09:14:21.048815742Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:21.048842065Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:14:21.628699891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:14:26.629145736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:14:27.057580041Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:27.057602963Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:14:31.629133069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:14:36.628666141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:14:37.153360514Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:14:37.153385055Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-17T09:14:37.194900073Z caller=logging.go:137 msg="GET /ready (503) 95.876µs"
level=warn ts=2026-04-17T09:14:41.628745937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:14:46.628878097Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-17T09:14:46.628909776Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-17T09:14:46.6289165Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:14:46.628934527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:14:47.192962053Z caller=logging.go:137 msg="GET /ready (503) 79.245µs"
level=warn ts=2026-04-17T09:14:51.629054919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:14:55.907673799Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:14:56.628937234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:15:01.629481413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:15:02.393394536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:54796"
ts=2026-04-17T09:15:04.553771022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:58538"
level=warn ts=2026-04-17T09:15:06.628523759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:15:07.002545423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
ts=2026-04-17T09:15:08.069596661Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.86:7946: dial tcp 10.129.0.86:7946: connect: connection refused"
ts=2026-04-17T09:15:08.069726923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.90:7946"
ts=2026-04-17T09:15:08.069745701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.90:43078"
ts=2026-04-17T09:15:08.070361906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.85:7946"
ts=2026-04-17T09:15:08.071867801Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.91:7946: dial tcp 10.129.0.91:7946: connect: connection refused"
ts=2026-04-17T09:15:08.072735832Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.89:7946: dial tcp 10.129.0.89:7946: connect: connection refused"
ts=2026-04-17T09:15:08.07291259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.92:7946"
ts=2026-04-17T09:15:08.073499796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.87:7946"
level=info ts=2026-04-17T09:15:08.073906427Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=51.461663465s
level=warn ts=2026-04-17T09:15:11.629343147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:15:13.304582331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:35248"
level=info ts=2026-04-17T09:15:16.610362465Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:15:16.6105413Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-17T09:15:16.610604869Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1973796 n=1
level=debug ts=2026-04-17T09:15:16.610664354Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1973796 n=1
level=debug ts=2026-04-17T09:15:16.610745837Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:15:16.629052584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:15:16.629090236Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:15:16.629101687Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:15:16.630100253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:15:19.119098659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:15:21.628515067Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:15:26.629068811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:15:30.217943311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:15:30.218004673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:15:30.224060575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:15:30.518779572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:15:30.525124955Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:15:31.628693352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:15:36.629050837Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:15:37.00370739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:15:39.424385552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:15:41.62886625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:15:46.309760106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:15:46.313308354Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:15:46.629375268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:15:46.629387446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:15:46.629410372Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:15:46.629420354Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:15:48.269302793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:15:48.276303266Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:15:51.629474199Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:15:56.629286569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:15:58.644130945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:16:01.629221785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:16:04.193971263Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-17T09:16:04.556332297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40046"
level=warn ts=2026-04-17T09:16:06.628527987Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:16:07.004793013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=warn ts=2026-04-17T09:16:11.629286912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:16:12.756336833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:16:12.760127966Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:16:16.60968259Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:16:16.610731786Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-17T09:16:16.628586074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:16:16.628588936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:16:16.628608509Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:16:16.628616488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:16:21.629123762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:16:23.887374127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T09:16:26.453243636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:16:26.460219611Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:16:26.629123192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:16:27.384825655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:50200"
level=debug ts=2026-04-17T09:16:28.954921401Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:16:31.62863885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:16:34.557124365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:34592"
level=warn ts=2026-04-17T09:16:36.628623707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:16:37.005490393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=warn ts=2026-04-17T09:16:41.628874234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:16:46.628800782Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:16:46.628826955Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:16:46.628834996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:16:46.628838621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:16:51.628789391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:16:53.482050386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:16:53.487303652Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:16:54.234309922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T09:16:54.23596674Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:16:56.628728583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:16:57.386035548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:33968"
level=warn ts=2026-04-17T09:17:01.629364011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:17:03.146628983Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:17:06.629421861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:17:07.006745939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=warn ts=2026-04-17T09:17:11.628513932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:17:13.30993644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:44194"
level=info ts=2026-04-17T09:17:16.610081815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:17:16.610093365Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:17:16.628467203Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:17:16.628484256Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:17:16.628491959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:17:16.628486203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:17:18.369249928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:17:18.377244294Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:17:21.629109601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:17:26.6290495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:17:27.386707025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:60828"
level=debug ts=2026-04-17T09:17:28.116041543Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:17:31.629361929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:17:34.564199753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:17:34.56765755Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:17:36.628550576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:17:37.008257834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:17:40.490920138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:17:41.628764187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:17:46.629099767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:17:46.629103141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:17:46.629123666Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:17:46.629132028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:17:51.629118678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:17:52.924589478Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:17:56.629459059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:18:01.629240595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:18:02.846927994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:18:02.856133275Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-17T09:18:04.561619601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:48568"
level=warn ts=2026-04-17T09:18:06.629356091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:18:07.009679604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=warn ts=2026-04-17T09:18:11.628930073Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:18:16.610200402Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:18:16.610264006Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:18:16.629169239Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:18:16.629174975Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:18:16.629186046Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:18:16.629195278Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:18:19.940492319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:18:19.945270921Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:18:20.47665641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:18:21.62916577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:18:26.628606787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:18:31.629280258Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:18:36.628828481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:18:37.010642409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=warn ts=2026-04-17T09:18:41.62897133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:18:46.093369746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T09:18:46.629421683Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:18:46.629445354Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:18:46.629454275Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:18:46.629446168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:18:46.887066387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:18:46.894667226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:18:48.235356957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:18:51.628869809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:18:56.62919272Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:19:01.628823802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:19:04.564598035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:51464"
level=warn ts=2026-04-17T09:19:06.629189094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:19:07.011807331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=warn ts=2026-04-17T09:19:11.629144978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:19:15.958893457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:19:15.965394763Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:19:16.609392463Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:19:16.610478364Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:19:16.629033807Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-17T09:19:16.629066998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:19:16.629071446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:19:16.629090204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:19:16.629097561Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:19:16.62916541Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-04-17T09:19:16.629353978Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T09:19:20.984242856Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-17T09:19:21.62886644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:19:21.935219807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:19:21.935242964Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-17T09:19:21.961655893Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:19:25.339871368Z caller=checkpoint.go:378 msg="writing series" size="211 kB"
level=warn ts=2026-04-17T09:19:26.629426901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:19:29.694501698Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-17T09:19:31.628854441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:19:33.173608844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:19:33.180046692Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:19:34.049587276Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
level=warn ts=2026-04-17T09:19:36.628644732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:19:36.822587184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:19:36.829447768Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-17T09:19:37.013247614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:19:38.403718269Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:19:41.629416823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:19:42.759250336Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=info ts=2026-04-17T09:19:46.629157024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:19:46.629182094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:19:46.629192513Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:19:46.629197141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:19:47.113872264Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
level=debug ts=2026-04-17T09:19:51.468431006Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=warn ts=2026-04-17T09:19:51.628659363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:19:52.633487187Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:19:55.822993652Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=debug ts=2026-04-17T09:19:55.826899185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-17T09:19:56.628947439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:19:57.391834887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:41452"
level=info ts=2026-04-17T09:19:57.675051579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:19:57.678813053Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:20:00.178181423Z caller=checkpoint.go:378 msg="writing series" size="311 kB"
level=info ts=2026-04-17T09:20:00.3295097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:20:00.333076128Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:20:01.629208541Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:04.533234773Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=warn ts=2026-04-17T09:20:06.629243781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:20:07.014473552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:20:08.888357295Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=warn ts=2026-04-17T09:20:11.628929843Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:13.243096206Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
ts=2026-04-17T09:20:13.31698787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:48512"
level=info ts=2026-04-17T09:20:16.610153961Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:20:16.610565582Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:20:16.629453388Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:20:16.62947072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:20:16.629479139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:20:16.629496189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:17.597742114Z caller=checkpoint.go:378 msg="writing series" size="212 kB"
level=debug ts=2026-04-17T09:20:17.597765274Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=15
level=debug ts=2026-04-17T09:20:19.462616313Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:20:21.199460521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:20:21.206754039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:20:21.628556978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:21.952984133Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-17T09:20:26.307589414Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:20:26.629424371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:20:27.393427631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:51890"
level=debug ts=2026-04-17T09:20:30.662053022Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-17T09:20:31.628768782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:35.017361193Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-17T09:20:36.629035516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:20:37.015444122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:20:39.372155447Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=info ts=2026-04-17T09:20:40.249851507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:20:40.254902995Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:20:41.628990239Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:43.726264791Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=debug ts=2026-04-17T09:20:46.405117374Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:20:46.628473132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:20:46.62849297Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:20:46.628500998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:20:46.628518308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:48.081075163Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-17T09:20:49.415118361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-17T09:20:50.496928235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:20:50.50113172Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:20:51.629108161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:52.436742442Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-17T09:20:56.628550993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:20:56.790908172Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-17T09:20:57.394359269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:47308"
level=debug ts=2026-04-17T09:21:01.14605834Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-04-17T09:21:01.62924693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:21:04.569055041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:32970"
level=debug ts=2026-04-17T09:21:05.501297798Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T09:21:06.629184984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:21:07.016653864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:21:09.85578726Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-17T09:21:11.628602926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:21:14.214747335Z caller=checkpoint.go:378 msg="writing series" size="5.5 MB"
level=debug ts=2026-04-17T09:21:14.214777373Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.9 MB" series=13
level=info ts=2026-04-17T09:21:16.610215439Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:21:16.610685629Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=warn ts=2026-04-17T09:21:16.629122685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:21:16.629129705Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:21:16.629146133Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:21:16.629153676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:21:18.56586713Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-17T09:21:19.9817688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:21:19.991266737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:21:20.939105121Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:21:21.628660219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:21:22.920515577Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-17T09:21:26.629070438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:21:27.274834544Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=warn ts=2026-04-17T09:21:31.628849422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:21:31.629987797Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=debug ts=2026-04-17T09:21:35.984693775Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-17T09:21:36.628969235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:21:37.017464825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T09:21:39.36255644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:21:39.3665713Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:21:39.366592641Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:21:39.366614364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:21:39.369797218Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:21:40.339944269Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-17T09:21:41.629240466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:21:44.694708541Z caller=checkpoint.go:378 msg="writing series" size="325 kB"
level=info ts=2026-04-17T09:21:46.629104591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:21:46.629112259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:21:46.629124605Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:21:46.629131993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:21:46.962178095Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:21:49.049753919Z caller=checkpoint.go:378 msg="writing series" size="6.6 kB"
level=info ts=2026-04-17T09:21:49.722617944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:21:49.727269963Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:21:50.730873859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-17T09:21:50.73297897Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:21:51.62954385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:21:53.403739083Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-17T09:21:56.268712614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:21:56.273131318Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:21:56.628686769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:21:57.759085627Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=debug ts=2026-04-17T09:21:57.759108695Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-17T09:22:01.629053776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:22:02.114069184Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=info ts=2026-04-17T09:22:02.313700009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:22:02.320460033Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:22:02.320478517Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:22:02.320502504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:22:02.326088648Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-17T09:22:04.571158358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39184"
level=debug ts=2026-04-17T09:22:06.468655216Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
level=warn ts=2026-04-17T09:22:06.629419024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:22:07.018209599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:22:10.823186352Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=warn ts=2026-04-17T09:22:11.629145635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:22:13.321980248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:41712"
level=debug ts=2026-04-17T09:22:15.177953152Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=info ts=2026-04-17T09:22:16.610392611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:22:16.610832051Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:22:16.629113908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:22:16.6291181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:22:16.629130447Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:22:16.629138153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:22:17.071670279Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:22:19.533510318Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=info ts=2026-04-17T09:22:21.085763897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:22:21.092868583Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:22:21.62906021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:22:23.887902299Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-17T09:22:24.180740603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:22:24.184875359Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:22:26.629243607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:22:28.242997458Z caller=checkpoint.go:378 msg="writing series" size="449 kB"
level=debug ts=2026-04-17T09:22:28.243041695Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-04-17T09:22:28.761973819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:22:31.628943647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:22:32.597350225Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=warn ts=2026-04-17T09:22:36.628863148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:22:36.952171301Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
ts=2026-04-17T09:22:37.019190121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T09:22:41.279583584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:22:41.284289863Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:22:41.307474786Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:22:41.628501657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:22:45.280109347Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:22:45.662966106Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-17T09:22:45.662990776Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-04-17T09:22:46.628974367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:22:46.628999713Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:22:46.62902832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:22:46.629023285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:22:47.495426882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:22:47.502429055Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:22:50.01707857Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-17T09:22:51.62864229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:22:54.371617841Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-17T09:22:56.629234279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:22:57.399360407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:49212"
level=debug ts=2026-04-17T09:22:58.726150435Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-17T09:23:01.62874723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:23:03.081616216Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
ts=2026-04-17T09:23:04.574181632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:46654"
level=warn ts=2026-04-17T09:23:06.629185089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:23:07.020320487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:23:07.436665976Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=info ts=2026-04-17T09:23:10.046885856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:23:10.051730147Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:23:11.603382172Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:23:11.629432174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:23:11.79082424Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=debug ts=2026-04-17T09:23:16.146536433Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=debug ts=2026-04-17T09:23:16.609004217Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:23:16.610034101Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-17T09:23:16.628901145Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:23:16.62891264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:23:16.628932171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:23:16.628939783Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:23:20.500822979Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=warn ts=2026-04-17T09:23:21.629491498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:23:24.855605702Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-17T09:23:26.629209121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:23:27.400230234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:50232"
level=debug ts=2026-04-17T09:23:28.524191631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:23:29.210399047Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-17T09:23:31.628808207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:23:33.565098227Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
level=info ts=2026-04-17T09:23:34.544744748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:23:34.548598682Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:23:36.629416383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:23:37.021168359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=info ts=2026-04-17T09:23:37.778506834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:23:37.785003282Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:23:37.920491464Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-17T09:23:41.628859107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:23:42.274740957Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-04-17T09:23:46.000950563Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:23:46.628956552Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:23:46.628984573Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-17T09:23:46.628981205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:23:46.62899614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:23:46.630039864Z caller=checkpoint.go:390 msg="flushing series" totalSize="480 kB" series=13
level=info ts=2026-04-17T09:23:46.634428712Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-17T09:23:46.637565799Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008260594s
level=warn ts=2026-04-17T09:23:51.628820792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:23:52.302432781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:23:52.309338055Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:23:56.629185599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:24:01.629485913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:24:04.576838651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:49104"
level=debug ts=2026-04-17T09:24:04.624317917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:24:06.628515988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:24:07.022821249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=warn ts=2026-04-17T09:24:11.62852093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:24:16.610206063Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:24:16.610253708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:24:16.628814475Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-17T09:24:16.628821882Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:24:16.628841263Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:24:16.628851907Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:24:16.628847532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:24:16.62894382Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-04-17T09:24:16.629153147Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T09:24:20.984080741Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-17T09:24:21.007523725Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:24:21.629061105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:24:25.339247474Z caller=checkpoint.go:378 msg="writing series" size="211 kB"
level=warn ts=2026-04-17T09:24:26.62875851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:24:27.402759787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:48538"
level=info ts=2026-04-17T09:24:28.220515329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:24:28.225466296Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:24:29.694455828Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-17T09:24:31.629207556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:24:34.049065654Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
level=info ts=2026-04-17T09:24:34.052144838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:24:34.058454652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:24:36.629062755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:24:37.024542735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:24:38.403758121Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:24:41.629128605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:24:42.758661081Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-04-17T09:24:46.62943202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:24:46.629437841Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:24:46.629459171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:24:46.629467747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:24:47.113186092Z caller=checkpoint.go:378 msg="writing series" size="248 kB"
level=debug ts=2026-04-17T09:24:49.279645243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:24:50.449861659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:24:50.454668277Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:24:51.468332303Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=warn ts=2026-04-17T09:24:51.62899502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:24:55.823366634Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-17T09:24:56.628936388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:00.178149109Z caller=checkpoint.go:378 msg="writing series" size="341 kB"
level=warn ts=2026-04-17T09:25:01.629052125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:01.952374274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:25:04.532419692Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
ts=2026-04-17T09:25:04.579216392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:34454"
level=warn ts=2026-04-17T09:25:06.628678281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:25:07.025415698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T09:25:08.28638207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:25:08.290049906Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:25:08.887418757Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=debug ts=2026-04-17T09:25:08.887442066Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=13
level=warn ts=2026-04-17T09:25:11.628756972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:13.242706605Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=debug ts=2026-04-17T09:25:16.609486003Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:25:16.610562281Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:25:16.628768257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:25:16.628787254Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:25:16.628795013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:25:16.628790073Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:17.59793363Z caller=checkpoint.go:378 msg="writing series" size="353 kB"
level=warn ts=2026-04-17T09:25:21.62917551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:21.952671821Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-17T09:25:22.305611759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:25:26.307097273Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:25:26.628630455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:25:29.96039737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:25:29.967599648Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:25:30.661426312Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-17T09:25:31.629189909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:35.016944103Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-04-17T09:25:36.629232076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:25:37.026704645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T09:25:37.337767752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:25:37.342128257Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:25:39.371875384Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-04-17T09:25:41.629059612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:43.726238216Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=info ts=2026-04-17T09:25:46.628620184Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:25:46.628624217Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:46.628647413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:25:46.628658309Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:25:46.880311992Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:25:48.081241082Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-17T09:25:51.629053038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:52.436041023Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-04-17T09:25:55.880041402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:25:55.884860432Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:25:55.884881759Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:25:55.884906953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:25:55.888554971Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:25:56.62873355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:25:56.791060924Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-17T09:25:59.688545703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:25:59.690553882Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-17T09:26:01.145563015Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-04-17T09:26:01.629106323Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:05.500428633Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T09:26:06.628816117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:26:07.027962704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T09:26:08.574276932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:26:08.582414373Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:26:09.855035635Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-17T09:26:11.629518707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:13.097981607Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:26:14.21689077Z caller=checkpoint.go:378 msg="writing series" size="6.2 MB"
level=debug ts=2026-04-17T09:26:14.216912997Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.1 MB" series=15
level=info ts=2026-04-17T09:26:16.193771836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:26:16.201345723Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:26:16.609539334Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:26:16.610605131Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:26:16.629354033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:26:16.629368828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:26:16.629376598Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:26:16.629393903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:18.565149162Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-17T09:26:21.628908652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:22.920180065Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-04-17T09:26:24.501190529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:26:24.505165664Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:26:26.628994328Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:27.275258458Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=warn ts=2026-04-17T09:26:31.6287289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:31.629880895Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
ts=2026-04-17T09:26:34.582589722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56284"
level=debug ts=2026-04-17T09:26:35.984796426Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-17T09:26:36.486233931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:26:36.492294702Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:26:36.628741752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:26:37.029440588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:26:40.339703747Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-17T09:26:41.628934229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:44.694448943Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=debug ts=2026-04-17T09:26:45.857645086Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:26:46.629059158Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:26:46.629082617Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:26:46.629090659Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:26:46.629082873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:49.048826641Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=debug ts=2026-04-17T09:26:51.451592578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:26:51.62883932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:53.403746549Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-17T09:26:56.629441387Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:26:57.758704182Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-17T09:27:01.628856131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:27:02.114074283Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=debug ts=2026-04-17T09:27:06.468856996Z caller=checkpoint.go:378 msg="writing series" size="343 kB"
level=debug ts=2026-04-17T09:27:06.468880254Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=12
level=warn ts=2026-04-17T09:27:06.629051676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:27:07.031123196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:27:10.82323101Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=warn ts=2026-04-17T09:27:11.629397396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:27:14.58205739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:27:14.590206212Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:27:15.177909017Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=debug ts=2026-04-17T09:27:15.814778558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:27:16.609674073Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:27:16.610753538Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:27:16.628479632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:27:16.628497417Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:27:16.628506387Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:27:16.628512913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:27:17.840527619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:27:17.846256379Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:27:19.532428368Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-17T09:27:21.629145917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:27:23.887581137Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-17T09:27:24.162244718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:27:24.166366076Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:27:26.62893267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:27:28.242912476Z caller=checkpoint.go:378 msg="writing series" size="265 kB"
level=warn ts=2026-04-17T09:27:31.628807939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:27:32.59730086Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-17T09:27:36.628519414Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:27:36.952663488Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
ts=2026-04-17T09:27:37.032945859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:27:41.307246517Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:27:41.628830879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:27:43.335970466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:52582"
level=debug ts=2026-04-17T09:27:45.662703275Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-17T09:27:45.662729649Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=9
level=debug ts=2026-04-17T09:27:45.810465172Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:27:46.629473785Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:27:46.629470447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:27:46.629496003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:27:46.629504612Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:27:50.016414075Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T09:27:51.628943407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:27:53.609268495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:27:54.37208862Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-17T09:27:56.629113525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:27:57.409637799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:33216"
level=info ts=2026-04-17T09:27:58.409883401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:27:58.413727741Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:27:58.7266911Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-17T09:28:01.629022399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:28:03.080850943Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-04-17T09:28:04.785471231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:28:04.79213583Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:28:06.628959101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:28:07.033774227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:28:07.436320802Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-17T09:28:11.628554182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:28:11.790990239Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=info ts=2026-04-17T09:28:13.814989653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:28:13.818819028Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:28:13.818840747Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:28:13.818861755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:28:13.821845052Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:28:15.764920754Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:28:16.146067508Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=debug ts=2026-04-17T09:28:16.609399292Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:28:16.610451091Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:28:16.629058477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:28:16.629074404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:28:16.629083015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:28:16.629081399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:28:20.50107277Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-17T09:28:21.62863146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:28:24.855944071Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-17T09:28:25.527683986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:28:25.533324137Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:28:26.628554817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:28:29.210578363Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=debug ts=2026-04-17T09:28:29.789504909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:28:31.628848482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:28:33.565573112Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-17T09:28:36.629069737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:28:37.035178266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:28:37.920130904Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-17T09:28:38.910729622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:28:38.916088441Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:28:41.629107831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:28:42.274955191Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-17T09:28:46.629387514Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:28:46.62939685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:28:46.629413863Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:28:46.629422636Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:28:46.629425839Z caller=checkpoint.go:390 msg="flushing series" totalSize="527 kB" series=13
level=info ts=2026-04-17T09:28:46.634650223Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-17T09:28:46.636055884Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006940307s
level=debug ts=2026-04-17T09:28:51.569817453Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:28:51.629148495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:28:56.628989534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:28:57.411202857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:51584"
level=info ts=2026-04-17T09:29:00.924197607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:29:00.928766954Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:29:01.62908475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:29:02.266866012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:29:02.273735632Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:29:02.273756408Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:29:02.27378138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:29:02.279266755Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:29:06.628505201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:29:07.035901672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=warn ts=2026-04-17T09:29:11.628936444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:29:13.840861406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:29:13.85014103Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:29:16.60922132Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:29:16.610287167Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-17T09:29:16.628779383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:29:16.628786856Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:29:16.628805439Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:29:16.628813452Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:29:16.628809001Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-17T09:29:16.628937648Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-17T09:29:16.629129145Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T09:29:17.279169706Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:29:20.115087086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:29:20.984086321Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-17T09:29:21.629468263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:29:25.33982621Z caller=checkpoint.go:378 msg="writing series" size="212 kB"
level=warn ts=2026-04-17T09:29:26.629025144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:29:27.412557054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:45628"
level=debug ts=2026-04-17T09:29:29.694066809Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-17T09:29:31.629109496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:29:34.049397217Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
level=warn ts=2026-04-17T09:29:36.629446612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:29:37.037229505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T09:29:37.776225708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:29:37.780814404Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:29:38.403715125Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:29:41.62911957Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:29:42.711715456Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:29:42.75892977Z caller=checkpoint.go:378 msg="writing series" size="155 kB"
ts=2026-04-17T09:29:43.341150038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:52892"
level=info ts=2026-04-17T09:29:44.229282903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:29:44.2355498Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-17T09:29:46.629380401Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:29:46.629388242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:29:46.629404748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:29:46.629412544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:29:47.113362284Z caller=checkpoint.go:378 msg="writing series" size="337 kB"
level=debug ts=2026-04-17T09:29:51.468155339Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=warn ts=2026-04-17T09:29:51.628913171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:29:55.329519359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:29:55.343728593Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:29:55.823238656Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-17T09:29:56.629185184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:00.178398139Z caller=checkpoint.go:378 msg="writing series" size="341 kB"
level=debug ts=2026-04-17T09:30:00.178427384Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-17T09:30:01.629048246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:04.532577021Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=warn ts=2026-04-17T09:30:06.628861575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:30:07.038722915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:30:08.88752549Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=debug ts=2026-04-17T09:30:11.351620506Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:30:11.629078001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:12.128962101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:30:12.130720199Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-17T09:30:13.242903296Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=debug ts=2026-04-17T09:30:16.60990058Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:30:16.61096604Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:30:16.611170168Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-17T09:30:16.611211656Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1973797 n=1
level=debug ts=2026-04-17T09:30:16.611258483Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1973797 n=1
level=info ts=2026-04-17T09:30:16.62911234Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:30:16.62911409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:16.629127584Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:30:16.629135328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:30:17.597089Z caller=checkpoint.go:378 msg="writing series" size="256 kB"
level=warn ts=2026-04-17T09:30:21.629426243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:21.95223348Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-17T09:30:26.307074622Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:30:26.628712604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:30.661709475Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-17T09:30:31.628704405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:35.016445382Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-04-17T09:30:36.6293232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:30:37.039830068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T09:30:37.495933538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:30:37.502811792Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:30:39.371622813Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-17T09:30:41.629086571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:30:43.154895382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:30:43.161268916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:30:43.726142964Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=debug ts=2026-04-17T09:30:45.074405853Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:30:46.629022215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:30:46.62902524Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:30:46.629068968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:30:46.6290766Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:30:47.573222544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:30:47.57719029Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:30:48.081655989Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-17T09:30:48.081680856Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-17T09:30:51.629163343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:52.436397152Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-17T09:30:56.628737128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:30:56.791067741Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-17T09:30:59.839240567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:31:01.145519253Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-04-17T09:31:01.629239029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:05.50069419Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T09:31:06.628503802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:31:07.041442179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:31:09.854954504Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-17T09:31:11.628712649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:13.891409386Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:31:14.214960487Z caller=checkpoint.go:378 msg="writing series" size="6.8 MB"
level=debug ts=2026-04-17T09:31:14.214984516Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.9 MB" series=6
level=debug ts=2026-04-17T09:31:16.609886246Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:31:16.610945555Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:31:16.629303346Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:31:16.629320819Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:31:16.629329203Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:31:16.629325583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:18.565072298Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-17T09:31:21.629255079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:22.91998077Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-17T09:31:26.629190098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:27.275198587Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=info ts=2026-04-17T09:31:27.68235565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:31:27.687752356Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:31:31.629054495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:31.630182944Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=debug ts=2026-04-17T09:31:35.984418432Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-17T09:31:36.628738167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:31:37.042877453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:31:40.339152595Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-04-17T09:31:41.628807068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:31:42.964114947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:31:42.971584911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:31:44.323081214Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:31:44.694196974Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=info ts=2026-04-17T09:31:46.628706208Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:31:46.62873354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:46.628742157Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:31:46.628756313Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:31:47.067738868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:31:47.071617447Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:31:49.049372805Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-17T09:31:49.764947913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:31:51.629081891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:53.403796821Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-17T09:31:56.628559704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:31:57.759269168Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=debug ts=2026-04-17T09:31:57.759293515Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-17T09:32:01.6291184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:02.114068052Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-17T09:32:04.596630129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39008"
level=debug ts=2026-04-17T09:32:06.468425127Z caller=checkpoint.go:378 msg="writing series" size="432 kB"
level=warn ts=2026-04-17T09:32:06.628750386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:32:07.043958626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:32:09.931755876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:32:10.823334779Z caller=checkpoint.go:378 msg="writing series" size="316 kB"
level=warn ts=2026-04-17T09:32:11.629473793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:15.178307889Z caller=checkpoint.go:378 msg="writing series" size="149 kB"
level=info ts=2026-04-17T09:32:16.610076784Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:32:16.610476952Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:32:16.628755472Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:32:16.628772278Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:32:16.628780065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:32:16.628781362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:19.533208858Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-17T09:32:21.629123001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:23.88761816Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-17T09:32:26.437814766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:32:26.441068869Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:32:26.441089144Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:32:26.441110331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:32:26.444068117Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:32:26.629180555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:28.066253372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:32:28.242906014Z caller=checkpoint.go:378 msg="writing series" size="272 kB"
level=debug ts=2026-04-17T09:32:28.242924016Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-04-17T09:32:31.629216745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:32.597617528Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=info ts=2026-04-17T09:32:34.984056433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:32:34.997391224Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:32:36.629428752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:36.952383866Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
ts=2026-04-17T09:32:37.045197168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T09:32:37.570419058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:32:37.57441869Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:32:38.483108682Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:32:38.878359908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:32:38.882175261Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:32:41.307133072Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:32:41.62878726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:32:43.350167408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:35534"
level=debug ts=2026-04-17T09:32:45.662436081Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-17T09:32:45.662460298Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-04-17T09:32:46.628622453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:32:46.628652Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:32:46.628660891Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:32:46.628666679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:50.017085969Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-17T09:32:51.628866242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:54.371625803Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-17T09:32:56.629022732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:32:58.726849601Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-17T09:33:01.62949141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:33:03.080843175Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:33:06.629207265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:33:07.046021477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:33:07.436301101Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=warn ts=2026-04-17T09:33:11.629066546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:33:11.79075753Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=debug ts=2026-04-17T09:33:12.967878167Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:33:13.913195793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-17T09:33:14.335274692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:33:14.346416169Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:33:16.146250403Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=debug ts=2026-04-17T09:33:16.609566824Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:33:16.61063581Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:33:16.62847282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:33:16.628500823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:33:16.628511686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:33:16.628528693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:33:20.501284904Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=warn ts=2026-04-17T09:33:21.629052875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:33:24.855895969Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-17T09:33:26.628914754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:33:27.422964563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:56572"
level=debug ts=2026-04-17T09:33:29.210072687Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-04-17T09:33:30.675051441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:33:30.682304314Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:33:31.628607148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:33:33.565530159Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-17T09:33:35.903685549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:33:35.907682324Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:33:36.628965536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:33:37.047090317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:33:37.647332836Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:33:37.920219811Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-17T09:33:41.628958463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:33:42.274955179Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-17T09:33:43.35324039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:40254"
level=info ts=2026-04-17T09:33:46.62910589Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:33:46.629135495Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-04-17T09:33:46.629125571Z caller=checkpoint.go:390 msg="flushing series" totalSize="579 kB" series=13
level=info ts=2026-04-17T09:33:46.629143884Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:33:46.629151765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:33:46.634490129Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-17T09:33:46.635843705Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006750644s
level=warn ts=2026-04-17T09:33:51.629350056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:33:55.422157818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:33:55.424071345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:33:56.628723051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:33:57.423908086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:59954"
level=info ts=2026-04-17T09:33:59.906976652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:33:59.910821568Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:34:01.629231571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:34:06.628747364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:34:07.047827991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:34:07.169653248Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:34:11.629320039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:34:16.610156097Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:34:16.610190929Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:34:16.629357327Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:34:16.629375967Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:34:16.629384275Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:34:16.629381398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:34:16.629389992Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-17T09:34:16.629511185Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-04-17T09:34:16.629679815Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T09:34:20.98547169Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-17T09:34:21.628915416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:34:22.458854019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:34:22.462985692Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:34:22.631836682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:34:22.638394159Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:34:25.339597807Z caller=checkpoint.go:378 msg="writing series" size="212 kB"
level=warn ts=2026-04-17T09:34:26.628966739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:34:27.425127788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:59762"
level=debug ts=2026-04-17T09:34:29.695131561Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-17T09:34:31.629378415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:34:33.946620059Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:34:34.049945177Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
ts=2026-04-17T09:34:34.602794216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50676"
level=warn ts=2026-04-17T09:34:36.628989568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:34:37.049497885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:34:38.404906783Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:34:41.62852256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:34:41.650672474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:34:41.656199567Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:34:42.759462379Z caller=checkpoint.go:378 msg="writing series" size="201 kB"
ts=2026-04-17T09:34:43.355251237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:56588"
level=info ts=2026-04-17T09:34:46.629269792Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:34:46.629268099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:34:46.629293748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:34:46.629301771Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:34:47.113937093Z caller=checkpoint.go:378 msg="writing series" size="217 kB"
level=debug ts=2026-04-17T09:34:49.339742882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:34:51.468596063Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-17T09:34:51.629058236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:34:55.823782693Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=warn ts=2026-04-17T09:34:56.629178069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:00.179152803Z caller=checkpoint.go:378 msg="writing series" size="341 kB"
level=debug ts=2026-04-17T09:35:00.179177618Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-04-17T09:35:00.360670939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:35:01.628849192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:04.533003209Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-04-17T09:35:05.247054391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:35:05.255386131Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:35:06.628528314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:35:07.050896809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:35:08.888452244Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-17T09:35:11.628951024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:13.242880672Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
ts=2026-04-17T09:35:13.356406722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:39954"
level=info ts=2026-04-17T09:35:16.610132791Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:35:16.61055565Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:35:16.628592194Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:35:16.628609316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:35:16.628619096Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:35:16.62862215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:17.597765232Z caller=checkpoint.go:378 msg="writing series" size="427 kB"
level=info ts=2026-04-17T09:35:19.752114217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:35:19.75808905Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:35:19.758108159Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:35:19.758130674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:35:19.761765499Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:35:21.629054921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:21.952432718Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-17T09:35:24.131503147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:35:26.307779636Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:35:26.628935078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:27.286300833Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-17T09:35:27.428125726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:44158"
level=debug ts=2026-04-17T09:35:30.663018317Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-17T09:35:31.629056756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:35:34.604915892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:39760"
level=debug ts=2026-04-17T09:35:35.017420274Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=warn ts=2026-04-17T09:35:36.629183806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:35:37.051893885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=info ts=2026-04-17T09:35:39.017399418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:35:39.021938204Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:35:39.371857591Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=info ts=2026-04-17T09:35:41.331375734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:35:41.335313241Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:35:41.629422265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:35:43.357668192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:47186"
level=debug ts=2026-04-17T09:35:43.727597187Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=debug ts=2026-04-17T09:35:43.727620467Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-04-17T09:35:46.628598295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:35:46.628621366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:35:46.628629953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:35:46.628635689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:48.081904265Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-17T09:35:51.62919395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:52.436144213Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-17T09:35:56.628500651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:35:56.791127965Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-17T09:36:01.146538689Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-04-17T09:36:01.62874333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:36:02.306748951Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:36:03.323446841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:36:03.330947463Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:36:03.330969454Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:36:03.330996406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:36:03.336830411Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:36:05.501152949Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T09:36:06.629102629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:36:07.05324564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:36:09.85579852Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-04-17T09:36:11.074025277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:36:11.078291371Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:36:11.629063594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:36:13.358965083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:53614"
level=debug ts=2026-04-17T09:36:14.216711647Z caller=checkpoint.go:378 msg="writing series" size="7.8 MB"
level=debug ts=2026-04-17T09:36:14.216734682Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.9 MB" series=7
level=debug ts=2026-04-17T09:36:16.609726138Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:36:16.610796779Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:36:16.629001384Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:36:16.629021084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:36:16.629033491Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:36:16.629041988Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:36:18.566204751Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-17T09:36:21.62892092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:36:22.015075284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:36:22.023350039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:36:22.573922106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:36:22.920211514Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-17T09:36:26.629230324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:36:27.275735748Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=info ts=2026-04-17T09:36:27.465258965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:36:27.468883226Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:36:28.088019154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:36:31.628514977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:36:31.630733092Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
ts=2026-04-17T09:36:34.606882683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:56346"
level=debug ts=2026-04-17T09:36:35.984711031Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-17T09:36:36.628972597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:36:37.054610427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:36:40.340216481Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-17T09:36:41.629073371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:36:44.694724829Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=warn ts=2026-04-17T09:36:46.628782256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:36:46.628777867Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:36:46.628806787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:36:46.628815095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:36:49.049798742Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-17T09:36:51.628848419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:36:53.388178137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:36:53.396582076Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:36:53.404728963Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-17T09:36:53.768747481Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:36:56.629502369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:36:57.758975372Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-04-17T09:36:57.758994467Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-04-17T09:37:01.62934196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:37:02.114621191Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=debug ts=2026-04-17T09:37:04.901678783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:37:06.469432393Z caller=checkpoint.go:378 msg="writing series" size="351 kB"
level=warn ts=2026-04-17T09:37:06.628830921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:37:07.056159923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T09:37:09.104548571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:37:09.108758451Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:37:10.780176486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:37:10.783941402Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:37:10.823371117Z caller=checkpoint.go:378 msg="writing series" size="334 kB"
level=warn ts=2026-04-17T09:37:11.62866054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:37:15.179168715Z caller=checkpoint.go:378 msg="writing series" size="179 kB"
level=debug ts=2026-04-17T09:37:16.609849443Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:37:16.610893027Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-17T09:37:16.628536375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:37:16.628537817Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:37:16.628554711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:37:16.628562851Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:37:19.533584563Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-17T09:37:21.628804431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:37:23.317163133Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:37:23.888771928Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-17T09:37:26.628915889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:37:28.243266746Z caller=checkpoint.go:378 msg="writing series" size="328 kB"
level=debug ts=2026-04-17T09:37:28.243291641Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-04-17T09:37:31.62892528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:37:32.59823137Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=info ts=2026-04-17T09:37:34.481506581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:37:34.490712832Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-17T09:37:34.609595958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:34448"
level=warn ts=2026-04-17T09:37:36.629374359Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:37:36.952501519Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-17T09:37:37.057189216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:37:41.308136031Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-17T09:37:41.628602754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:37:45.663860178Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-17T09:37:45.663884772Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-04-17T09:37:46.628570233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:37:46.628595385Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-17T09:37:46.628580873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:37:46.628603076Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:37:50.01710939Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
level=debug ts=2026-04-17T09:37:51.269763968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:37:51.271271108Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:37:51.628970589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:37:54.372173445Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=warn ts=2026-04-17T09:37:56.629364871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:37:58.270375555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:37:58.273987712Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:37:58.628188297Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:37:58.727482577Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-17T09:38:01.628840966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:38:01.648986732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:38:01.652906736Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:38:03.081829861Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-17T09:38:06.629463137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:38:07.058906004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:38:07.436406436Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-17T09:38:11.628712397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:38:11.791468343Z caller=checkpoint.go:378 msg="writing series" size="273 kB"
level=debug ts=2026-04-17T09:38:16.145995356Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-17T09:38:16.610075632Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:38:16.610079742Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=warn ts=2026-04-17T09:38:16.628596026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:38:16.628605837Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:38:16.628623593Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:38:16.628631836Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:38:20.501631437Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-17T09:38:21.629289892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:38:22.402362821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:38:22.409502621Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:38:24.300593726Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:38:24.856068038Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-17T09:38:26.629556534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:38:27.436108776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:40500"
level=debug ts=2026-04-17T09:38:29.211452987Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-17T09:38:31.628682288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:38:33.56622689Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-17T09:38:36.629234576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:38:37.060069149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:38:37.920824473Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-17T09:38:39.109056877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:38:39.112599627Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:38:39.112621622Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:38:39.112643737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:38:39.115619328Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:38:40.698454612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:38:41.628512007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:38:42.275000534Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-17T09:38:46.629046268Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:38:46.629048024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:38:46.62906843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:38:46.629075903Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:38:46.630137351Z caller=checkpoint.go:390 msg="flushing series" totalSize="705 kB" series=13
level=info ts=2026-04-17T09:38:46.635602572Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-17T09:38:46.637044695Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007404151s
level=warn ts=2026-04-17T09:38:51.629367728Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:38:53.984137809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:38:53.990478854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:38:55.77727525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:38:55.781240378Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:38:56.628830051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:38:57.437331591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:54112"
level=debug ts=2026-04-17T09:38:58.851603363Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:39:01.628751913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:39:06.629399454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:39:07.060804339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=warn ts=2026-04-17T09:39:11.629400386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:39:13.367033184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:50270"
level=debug ts=2026-04-17T09:39:16.609778406Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:39:16.610828673Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:39:16.628568296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:39:16.628585807Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:39:16.628594354Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:39:16.628603739Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-17T09:39:16.62861901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:39:16.628723876Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-04-17T09:39:16.628890108Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-04-17T09:39:16.716337841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:39:16.724309895Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:39:18.60097391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:39:20.720663713Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
level=warn ts=2026-04-17T09:39:21.628592881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:39:24.811560703Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-17T09:39:26.629186985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:39:27.43897515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:33258"
level=debug ts=2026-04-17T09:39:28.902501779Z caller=checkpoint.go:378 msg="writing series" size="241 kB"
level=debug ts=2026-04-17T09:39:29.125324108Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:39:31.629052598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:39:32.993151533Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-17T09:39:33.423491624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:39:33.428544899Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-17T09:39:34.614439092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:50866"
level=warn ts=2026-04-17T09:39:36.629256068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:39:37.062211025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:39:37.083888744Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=debug ts=2026-04-17T09:39:41.174823024Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:39:41.62875576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:39:45.266335847Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=info ts=2026-04-17T09:39:46.62918727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:39:46.629210201Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:39:46.629217994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:39:46.62921909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:39:49.357112664Z caller=checkpoint.go:378 msg="writing series" size="318 kB"
level=info ts=2026-04-17T09:39:51.50453785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:39:51.509723836Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:39:51.629225523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:39:53.447390986Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-17T09:39:56.012649871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:39:56.295275998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:39:56.629063416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:39:57.538533617Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-17T09:40:01.629135871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:01.629414629Z caller=checkpoint.go:378 msg="writing series" size="612 kB"
level=debug ts=2026-04-17T09:40:01.629431759Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=12
level=debug ts=2026-04-17T09:40:05.720397816Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=warn ts=2026-04-17T09:40:06.629109358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:40:07.063553591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=info ts=2026-04-17T09:40:09.74383328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:40:09.751413105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:40:09.811787064Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-17T09:40:11.629491478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:40:13.369647151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:39092"
level=debug ts=2026-04-17T09:40:13.90180487Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=debug ts=2026-04-17T09:40:16.609365465Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:40:16.610430118Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:40:16.629243676Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:40:16.629259402Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:40:16.629266958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:40:16.62926898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:17.992723916Z caller=checkpoint.go:378 msg="writing series" size="270 kB"
level=warn ts=2026-04-17T09:40:21.628943518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:22.083947679Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=debug ts=2026-04-17T09:40:23.532197089Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:40:23.961388089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:40:23.967476176Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:40:26.17507267Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:40:26.628981627Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:30.265909042Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-17T09:40:31.629030246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:34.356528464Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=info ts=2026-04-17T09:40:35.402280233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:40:35.409583554Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:40:36.628532062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:40:37.064221005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:40:38.447286567Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-04-17T09:40:41.486946662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:40:41.629277053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:42.538962646Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
ts=2026-04-17T09:40:43.371047719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:45702"
level=info ts=2026-04-17T09:40:46.62871287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:40:46.628737367Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:40:46.628746715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:40:46.628759645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:46.629877117Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=debug ts=2026-04-17T09:40:50.720086502Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-17T09:40:51.628508625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:52.434020676Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:40:52.434068436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:40:52.442615668Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:40:54.811247682Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-04-17T09:40:56.628929677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:40:58.90222623Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=warn ts=2026-04-17T09:41:01.628853115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:41:02.992857587Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-17T09:41:06.024351948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:41:06.036487323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:41:06.629291365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:41:07.065924171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:41:07.083750699Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-04-17T09:41:09.928763258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:41:09.936196219Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:41:11.182687034Z caller=checkpoint.go:378 msg="writing series" size="8.0 MB"
level=debug ts=2026-04-17T09:41:11.182710136Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.1 MB" series=17
level=warn ts=2026-04-17T09:41:11.629597534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:41:11.93985376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:41:11.945460588Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:41:15.265973926Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=debug ts=2026-04-17T09:41:16.609733443Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:41:16.610785998Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:41:16.628889061Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:41:16.62890751Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:41:16.628915434Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:41:16.628920564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:41:19.35667533Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-17T09:41:21.628673786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:41:23.448139077Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=debug ts=2026-04-17T09:41:24.472088834Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:41:24.890176075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:41:24.892424732Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:41:26.628986794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:41:27.53852495Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=info ts=2026-04-17T09:41:28.207648173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:41:28.212101017Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:41:31.628870414Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:41:31.628963721Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-17T09:41:34.620745015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:46788"
level=debug ts=2026-04-17T09:41:35.720775865Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=warn ts=2026-04-17T09:41:36.629100777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:41:37.067156776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T09:41:39.603757474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:41:39.616535328Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:41:39.811563953Z caller=checkpoint.go:378 msg="writing series" size="235 kB"
level=warn ts=2026-04-17T09:41:41.629130519Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:41:43.374192336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:45470"
level=debug ts=2026-04-17T09:41:43.902309317Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-17T09:41:46.629308675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:41:46.629321821Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:41:46.629349034Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:41:46.629361634Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:41:47.993157902Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-17T09:41:49.156748269Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:41:51.629157525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:41:52.083652555Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=debug ts=2026-04-17T09:41:52.083684528Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-04-17T09:41:56.175325501Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-17T09:41:56.628585642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:41:57.446292394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:32920"
level=info ts=2026-04-17T09:41:59.48508877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:41:59.496746983Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:42:00.26618728Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-17T09:42:01.629071078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:42:04.357610857Z caller=checkpoint.go:378 msg="writing series" size="668 kB"
level=warn ts=2026-04-17T09:42:06.62865551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:42:07.068284369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:42:08.44730018Z caller=checkpoint.go:378 msg="writing series" size="419 kB"
level=debug ts=2026-04-17T09:42:08.447325564Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-04-17T09:42:08.902540106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:42:08.908028261Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:42:11.629138028Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:42:12.538650356Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=debug ts=2026-04-17T09:42:16.609936306Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:42:16.610975679Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:42:16.628740391Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:42:16.628769873Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:42:16.628780792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:42:16.628778738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:42:16.629928646Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=debug ts=2026-04-17T09:42:20.720334151Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-17T09:42:20.73150184Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:42:21.628899942Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:42:22.254873803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-17T09:42:24.070406949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:42:24.075004785Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:42:24.075049083Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:42:24.075073755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:42:24.078591203Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:42:24.814519694Z caller=checkpoint.go:378 msg="writing series" size="4.6 MB"
level=debug ts=2026-04-17T09:42:24.814550198Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.8 MB" series=4
level=warn ts=2026-04-17T09:42:26.629430606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:42:28.460940322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:42:28.467570835Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:42:28.902200841Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=warn ts=2026-04-17T09:42:31.629235769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:42:32.993480908Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-17T09:42:34.624758237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:40494"
level=info ts=2026-04-17T09:42:36.000944606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:42:36.005729263Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:42:36.62864681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:42:37.069852866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:42:37.084505699Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-17T09:42:41.175364939Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-17T09:42:41.175387485Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=4
level=warn ts=2026-04-17T09:42:41.628920398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:42:43.376786437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:48442"
level=debug ts=2026-04-17T09:42:45.26598669Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-17T09:42:46.629002555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:42:46.62902523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:42:46.629079913Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:42:46.629091796Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:42:46.638202581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:42:46.646907274Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:42:46.646927074Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:42:46.646952012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:42:46.653786164Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:42:49.356509909Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=warn ts=2026-04-17T09:42:51.629137481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:42:52.348379906Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:42:53.447709082Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=warn ts=2026-04-17T09:42:56.629221178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:42:56.837633636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:42:56.842090346Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:42:57.538461398Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-04-17T09:42:59.870129498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:42:59.877494323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:43:01.629376929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:01.629400172Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=debug ts=2026-04-17T09:43:02.582047297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:43:05.720444205Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=warn ts=2026-04-17T09:43:06.62934907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:43:07.071616224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:43:09.811445543Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=warn ts=2026-04-17T09:43:11.629196035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:13.901830098Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-04-17T09:43:15.554271294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:43:15.559652504Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:43:15.812992191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:43:15.821598688Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:43:16.609390644Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:43:16.610423069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:43:16.629316047Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:43:16.629324096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:16.629339624Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:43:16.629350043Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:43:17.993597195Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-17T09:43:21.628941079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:22.084408219Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=debug ts=2026-04-17T09:43:26.174593956Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-17T09:43:26.423542314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:43:26.629033351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:30.265924121Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-04-17T09:43:30.852801793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:43:30.860210497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:43:31.628955833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:34.357109915Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-17T09:43:36.62859871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:43:37.073432989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:43:38.447444951Z caller=checkpoint.go:378 msg="writing series" size="6.6 kB"
level=warn ts=2026-04-17T09:43:41.628684687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:42.53850168Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-17T09:43:43.379955862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:53852"
level=debug ts=2026-04-17T09:43:46.62934154Z caller=checkpoint.go:390 msg="flushing series" totalSize="860 kB" series=15
level=info ts=2026-04-17T09:43:46.629376821Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:43:46.629363473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:46.629392179Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:43:46.629399823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:43:46.636251597Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-17T09:43:46.638793305Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009940885s
level=warn ts=2026-04-17T09:43:51.628708476Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:53.28935406Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:43:56.26860805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:43:56.274503249Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:43:56.628845425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:43:58.061616379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:44:01.62944915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:44:06.629526984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:44:07.074966401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T09:44:08.345821884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:44:08.354138957Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-17T09:44:08.77031325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:44:08.774469218Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:44:11.628599178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:44:12.758229834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:44:12.765537183Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-17T09:44:13.381352553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:44256"
level=debug ts=2026-04-17T09:44:16.60993194Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:44:16.610977349Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:44:16.628583645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:44:16.628613075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:44:16.62862352Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:44:16.628618532Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-17T09:44:16.628645538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:44:16.628781647Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-04-17T09:44:16.629000215Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T09:44:20.720203733Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=debug ts=2026-04-17T09:44:21.061214229Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:44:21.629269999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:44:24.811523836Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-17T09:44:26.629276171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:44:28.902811856Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=warn ts=2026-04-17T09:44:31.629045616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:44:32.993603329Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-17T09:44:36.629188344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:44:37.075938778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:44:37.083578347Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=info ts=2026-04-17T09:44:37.848054602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:44:37.853374713Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:44:41.174791781Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:44:41.629247395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:44:43.382999543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:36174"
level=debug ts=2026-04-17T09:44:45.265589143Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-17T09:44:46.628944146Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:44:46.628967214Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:44:46.628976395Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:44:46.628966827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:44:48.093261004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:44:49.356868455Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-04-17T09:44:51.130055718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:44:51.138078184Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:44:51.629061806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:44:53.448082033Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-17T09:44:54.001718561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:44:54.00561678Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:44:54.410665891Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:44:56.628572866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:44:57.539106426Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=warn ts=2026-04-17T09:45:01.629117305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:45:01.629406037Z caller=checkpoint.go:378 msg="writing series" size="562 kB"
level=debug ts=2026-04-17T09:45:01.629421133Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=12
ts=2026-04-17T09:45:04.632419492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:52248"
level=debug ts=2026-04-17T09:45:05.720689146Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-17T09:45:06.628923818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:45:07.076896235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T09:45:08.784067357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:45:08.795154901Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:45:09.811454773Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=warn ts=2026-04-17T09:45:11.628961949Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:45:13.902748822Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-04-17T09:45:16.609549259Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:45:16.610599648Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:45:16.610780054Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-17T09:45:16.610826147Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1973798 n=1
level=debug ts=2026-04-17T09:45:16.610867175Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1973798 n=1
level=info ts=2026-04-17T09:45:16.628905022Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:45:16.628925794Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:45:16.628934785Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:45:16.628928496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:45:17.993633822Z caller=checkpoint.go:378 msg="writing series" size="424 kB"
level=info ts=2026-04-17T09:45:18.004687782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:45:18.008863333Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:45:18.008882021Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:45:18.00890455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:45:18.012073694Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:45:21.232820909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:45:21.628888474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:45:22.084617345Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-04-17T09:45:26.174619388Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:45:26.62904379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:45:30.266382124Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-17T09:45:30.639537564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:45:30.646529413Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:45:31.62917671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:45:34.3571235Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=warn ts=2026-04-17T09:45:36.628749806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:45:37.078691107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:45:38.448267699Z caller=checkpoint.go:378 msg="writing series" size="341 kB"
level=debug ts=2026-04-17T09:45:38.448289506Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-04-17T09:45:41.208243904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:45:41.209987016Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:45:41.629475313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:45:42.538599825Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=info ts=2026-04-17T09:45:42.66305785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:45:42.670710876Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-17T09:45:43.386448098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:33670"
level=info ts=2026-04-17T09:45:46.420371127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:45:46.425182409Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:45:46.628609542Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:45:46.628639343Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-17T09:45:46.628635018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:45:46.628650541Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:45:46.629773393Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=debug ts=2026-04-17T09:45:48.439389229Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:45:50.72085269Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-17T09:45:51.629067813Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:45:54.811658543Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-04-17T09:45:56.629290035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:45:58.024059219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:45:58.031346168Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:45:58.902773323Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-04-17T09:46:01.629390454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:46:02.993075172Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
ts=2026-04-17T09:46:04.634855097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45212"
level=info ts=2026-04-17T09:46:05.356450057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:46:05.363167451Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:46:06.629373038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:46:07.080625917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:46:07.084403078Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=debug ts=2026-04-17T09:46:11.180776651Z caller=checkpoint.go:378 msg="writing series" size="8.7 MB"
level=debug ts=2026-04-17T09:46:11.180797829Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.1 MB" series=8
level=warn ts=2026-04-17T09:46:11.629095974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:46:12.972784686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-17T09:46:13.387505481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:54030"
level=debug ts=2026-04-17T09:46:15.266328278Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-17T09:46:16.610126972Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:46:16.610560657Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:46:16.629440282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:46:16.629458091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:46:16.629465929Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:46:16.629465764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:46:19.356514669Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-17T09:46:21.629043417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:46:23.447969227Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-17T09:46:26.629128949Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:46:27.45883614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:53028"
level=debug ts=2026-04-17T09:46:27.538497163Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=info ts=2026-04-17T09:46:30.196581271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:46:30.20431402Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:46:31.629285014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:46:31.629296349Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=info ts=2026-04-17T09:46:34.187646077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:46:34.192697891Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:46:35.720181601Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-17T09:46:36.629070474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:46:37.081756506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:46:39.811117475Z caller=checkpoint.go:378 msg="writing series" size="334 kB"
level=warn ts=2026-04-17T09:46:41.629511126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:46:42.977827148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:46:43.902210583Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=debug ts=2026-04-17T09:46:46.008196254Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:46:46.629368234Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:46:46.629392918Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:46:46.629401199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:46:46.629398589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:46:47.993619845Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-17T09:46:48.500139211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:46:48.505610751Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:46:51.629152459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:46:51.805447987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:46:51.81265919Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:46:52.084726876Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=debug ts=2026-04-17T09:46:52.084748836Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=debug ts=2026-04-17T09:46:56.17555713Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-04-17T09:46:56.628619172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:00.265431386Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=warn ts=2026-04-17T09:47:01.628545165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:04.357187255Z caller=checkpoint.go:378 msg="writing series" size="573 kB"
level=warn ts=2026-04-17T09:47:06.628558271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:47:07.083111371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:47:08.447657142Z caller=checkpoint.go:378 msg="writing series" size="532 kB"
level=debug ts=2026-04-17T09:47:08.447680464Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-04-17T09:47:11.628619455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:12.53843029Z caller=checkpoint.go:378 msg="writing series" size="539 kB"
level=debug ts=2026-04-17T09:47:16.609891924Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:47:16.610952525Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:47:16.629056414Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:47:16.629092919Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-04-17T09:47:16.62908709Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=info ts=2026-04-17T09:47:16.629103466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:47:16.62911117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:20.720083306Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-17T09:47:21.31048494Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:47:21.349563231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:47:21.355835402Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:47:21.629311755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:47:23.170248033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:47:23.176832808Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:47:23.176852766Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:47:23.176878329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:47:23.182437672Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:47:23.182451074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:47:24.816049481Z caller=checkpoint.go:378 msg="writing series" size="9.0 MB"
level=debug ts=2026-04-17T09:47:24.81607129Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.6 MB" series=4
level=warn ts=2026-04-17T09:47:26.628692597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:47:27.460865681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:52094"
level=debug ts=2026-04-17T09:47:28.902591468Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=info ts=2026-04-17T09:47:29.482933692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:47:29.487918427Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:47:31.629277506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:32.99318548Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-17T09:47:34.639066035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:59294"
level=info ts=2026-04-17T09:47:36.370566835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:47:36.377914935Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:47:36.629209924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:37.084531436Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-04-17T09:47:37.084646839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:47:41.175346296Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-17T09:47:41.175368104Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=4
level=warn ts=2026-04-17T09:47:41.628917905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:47:42.74486843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:47:42.754058812Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-17T09:47:43.392310574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:55154"
level=debug ts=2026-04-17T09:47:45.266408451Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-04-17T09:47:46.628476422Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:47:46.628500876Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:47:46.628509701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:47:46.628512833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:49.356684953Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=debug ts=2026-04-17T09:47:50.240974723Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:47:51.628682116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:53.447752761Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
level=warn ts=2026-04-17T09:47:56.629366571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:47:57.538823834Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-17T09:48:01.628523482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:48:01.629684994Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
ts=2026-04-17T09:48:04.640510136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33800"
level=debug ts=2026-04-17T09:48:05.720125501Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-04-17T09:48:06.359546285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:48:06.368420483Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:48:06.628532475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:48:07.085981878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:48:09.811254504Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=warn ts=2026-04-17T09:48:11.629150494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:48:13.904870894Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-17T09:48:13.9049021Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=8
level=debug ts=2026-04-17T09:48:15.028057661Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:48:16.610051349Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:48:16.61046134Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:48:16.629114348Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:48:16.629144016Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:48:16.629153774Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:48:16.629168655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:48:17.698130897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:48:17.704163816Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:48:17.992829398Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-17T09:48:20.040613187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:48:20.047586083Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-17T09:48:20.550998858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:48:20.555448918Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:48:21.629429345Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:48:22.084473649Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=debug ts=2026-04-17T09:48:25.449184602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:48:26.175153684Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-17T09:48:26.629287811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:48:30.265545839Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-17T09:48:31.628633534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:48:34.357094913Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-04-17T09:48:36.628519081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:48:37.08725309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:48:38.447798417Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-17T09:48:41.629167502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:48:42.538643931Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-17T09:48:43.394223945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:54170"
level=debug ts=2026-04-17T09:48:45.418823031Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:48:46.629123989Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:48:46.629158724Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:48:46.629171051Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:48:46.629160161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:48:46.629184Z caller=checkpoint.go:390 msg="flushing series" totalSize="205 kB" series=7
level=info ts=2026-04-17T09:48:46.636765563Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-17T09:48:46.639888975Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010940159s
level=warn ts=2026-04-17T09:48:51.628833822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T09:48:56.628970788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:48:57.465617654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:43444"
level=info ts=2026-04-17T09:48:59.274383051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:48:59.278065177Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:49:00.272221778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:49:01.629053909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:49:03.86786884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:49:03.874751951Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-17T09:49:06.069248956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:49:06.075601654Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:49:06.075619524Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:49:06.075641292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:49:06.081352122Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:49:06.62893449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:49:07.088510494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T09:49:08.305531461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:49:08.309648422Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:49:08.309671742Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:49:08.3096995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:49:08.313340461Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:49:11.628692915Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:49:16.609217487Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:49:16.610258894Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-17T09:49:16.628593102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:49:16.628598711Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:49:16.628619619Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:49:16.628624267Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-17T09:49:16.628633682Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:49:16.628747206Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-04-17T09:49:16.628956161Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=info ts=2026-04-17T09:49:19.873180881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:49:19.882604025Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:49:20.659251242Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=debug ts=2026-04-17T09:49:21.310090228Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:49:21.628520652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:49:22.983122212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:49:22.987200505Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:49:24.689146481Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-17T09:49:26.629469568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:49:28.719287853Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=warn ts=2026-04-17T09:49:31.628831319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:49:32.749189915Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
ts=2026-04-17T09:49:34.645101903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43524"
level=warn ts=2026-04-17T09:49:36.629068248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:49:36.778506068Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
ts=2026-04-17T09:49:37.090390233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:49:40.808269125Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:49:41.628664883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:49:44.512891674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:49:44.518967215Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:49:44.838165138Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=info ts=2026-04-17T09:49:46.629368566Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:49:46.629399035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:49:46.62940961Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:49:46.629457453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:49:47.103900549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:49:47.104994273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:49:47.108955146Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:49:47.111985271Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:49:48.868437059Z caller=checkpoint.go:378 msg="writing series" size="338 kB"
level=debug ts=2026-04-17T09:49:48.868464068Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-04-17T09:49:51.628664998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:49:52.029640285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:49:52.045290303Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:49:52.898059072Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-17T09:49:53.946693204Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:49:55.548540989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:49:55.550150248Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:49:56.628752072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:49:56.928171007Z caller=checkpoint.go:378 msg="writing series" size="571 kB"
ts=2026-04-17T09:49:57.467970164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:48706"
level=debug ts=2026-04-17T09:50:00.957893569Z caller=checkpoint.go:378 msg="writing series" size="373 kB"
level=warn ts=2026-04-17T09:50:01.629227082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:50:04.646731983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:37798"
level=debug ts=2026-04-17T09:50:04.988158509Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-17T09:50:06.628974391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:50:07.09123878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:50:09.01744569Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=debug ts=2026-04-17T09:50:09.017467371Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-04-17T09:50:11.629004466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:50:13.047656247Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-04-17T09:50:16.610273857Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:50:16.61073044Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:50:16.628771828Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:50:16.628794613Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:50:16.628806256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:50:16.628880485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:50:17.077828958Z caller=checkpoint.go:378 msg="writing series" size="303 kB"
level=debug ts=2026-04-17T09:50:21.107254468Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-17T09:50:21.629498145Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:50:25.136945902Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:50:26.628522613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:50:28.051586399Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:50:29.166580946Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-17T09:50:31.629242712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:50:33.196665119Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
ts=2026-04-17T09:50:34.648042384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:43824"
level=info ts=2026-04-17T09:50:35.135485147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:50:35.145727333Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-17T09:50:35.561787614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:50:35.566156419Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:50:36.6290572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:50:37.092995299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:50:37.22722048Z caller=checkpoint.go:378 msg="writing series" size="317 kB"
level=info ts=2026-04-17T09:50:39.054257876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:50:39.058090678Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:50:41.256286938Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=warn ts=2026-04-17T09:50:41.628985737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:50:43.399652554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:48892"
level=debug ts=2026-04-17T09:50:45.28637196Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-04-17T09:50:45.286397798Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-17T09:50:46.629345649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:50:46.629355793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:50:46.629375934Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:50:46.629383545Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:50:48.416880292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:50:48.424245461Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:50:49.315566605Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=warn ts=2026-04-17T09:50:51.628630621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:50:53.346371242Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=debug ts=2026-04-17T09:50:55.472794267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:50:56.628517018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:50:57.375564452Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=debug ts=2026-04-17T09:50:58.279002603Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:51:01.405265648Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T09:51:01.629267234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:51:05.435071948Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-17T09:51:06.628652893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:51:07.093979296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:51:09.475289792Z caller=checkpoint.go:378 msg="writing series" size="9.3 MB"
level=debug ts=2026-04-17T09:51:09.475316884Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.7 MB" series=6
level=warn ts=2026-04-17T09:51:11.628883153Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:51:13.400807628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:50326"
level=debug ts=2026-04-17T09:51:13.495468947Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-17T09:51:16.610086389Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:51:16.610105207Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:51:16.629219892Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:51:16.629240147Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:51:16.629249227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:51:16.629269513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:51:17.525361767Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-04-17T09:51:21.554628475Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-17T09:51:21.629067034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:51:24.12667214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:51:24.131034662Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:51:24.131053882Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:51:24.131074619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:51:24.134398869Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:51:25.584286895Z caller=checkpoint.go:378 msg="writing series" size="155 kB"
level=warn ts=2026-04-17T09:51:26.629179437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:51:29.615087909Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=info ts=2026-04-17T09:51:30.258757752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:51:30.264105667Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:51:30.97198012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:51:30.978375372Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:51:31.629112868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:51:33.208943509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:51:33.644343288Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-17T09:51:34.501132683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:51:36.628638883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:51:37.095072283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:51:37.674414213Z caller=checkpoint.go:378 msg="writing series" size="130 kB"
level=info ts=2026-04-17T09:51:40.094399309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:51:40.102719304Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:51:41.628939509Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:51:41.704354597Z caller=checkpoint.go:378 msg="writing series" size="273 kB"
level=debug ts=2026-04-17T09:51:41.704378009Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-17T09:51:43.630309335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:51:43.63436029Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:51:45.734301889Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-04-17T09:51:46.628470544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:51:46.628501356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-17T09:51:46.62850784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:51:46.6285177Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:51:49.763703284Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-17T09:51:51.628936521Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:51:53.794171606Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=warn ts=2026-04-17T09:51:56.629427887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:51:57.823642983Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=warn ts=2026-04-17T09:52:01.629500569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:52:01.853685603Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-04-17T09:52:03.363631253Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-17T09:52:04.652660844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:57216"
level=debug ts=2026-04-17T09:52:05.883966597Z caller=checkpoint.go:378 msg="writing series" size="629 kB"
level=debug ts=2026-04-17T09:52:05.883990924Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-04-17T09:52:06.628633535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:52:07.096779405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:52:09.913489343Z caller=checkpoint.go:378 msg="writing series" size="650 kB"
level=warn ts=2026-04-17T09:52:11.629212591Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:52:13.943580443Z caller=checkpoint.go:378 msg="writing series" size="711 kB"
level=debug ts=2026-04-17T09:52:13.943604241Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-04-17T09:52:14.320994749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:52:14.331443468Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:52:16.609681002Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:52:16.610705709Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:52:16.629453096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:52:16.629475303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:52:16.629486628Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:52:16.629508983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:52:17.616185065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:52:17.973038684Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=info ts=2026-04-17T09:52:21.142665807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:52:21.146723012Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:52:21.628996793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:52:22.002887574Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-17T09:52:24.126133126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:52:24.130736107Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:52:26.040181955Z caller=checkpoint.go:378 msg="writing series" size="13 MB"
level=debug ts=2026-04-17T09:52:26.040205393Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=3
level=warn ts=2026-04-17T09:52:26.630121011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:52:29.059998167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:52:29.067053221Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:52:30.062266163Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=warn ts=2026-04-17T09:52:31.628767776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:52:32.678474583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:52:34.092301208Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-17T09:52:34.654176683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:36518"
level=warn ts=2026-04-17T09:52:36.630836229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:52:37.098399301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:52:38.125195648Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-17T09:52:41.630002606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:52:42.152676403Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-17T09:52:42.152698632Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=4
level=debug ts=2026-04-17T09:52:46.182094785Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-17T09:52:46.630751382Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:52:46.630780967Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-17T09:52:46.630780768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:52:46.630791576Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:52:50.212154962Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-04-17T09:52:51.629182446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:52:54.241280441Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=warn ts=2026-04-17T09:52:56.630207941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:52:56.721095099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:52:56.739037078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:52:58.271742206Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=debug ts=2026-04-17T09:52:59.557100526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:53:01.628935582Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:53:02.30123892Z caller=checkpoint.go:378 msg="writing series" size="9.9 kB"
level=debug ts=2026-04-17T09:53:06.331683106Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=warn ts=2026-04-17T09:53:06.629266555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:53:07.100077011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:53:10.360728618Z caller=checkpoint.go:378 msg="writing series" size="357 kB"
level=warn ts=2026-04-17T09:53:11.629435897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:53:13.284259733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:53:13.289280087Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-17T09:53:13.413239363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:55652"
level=debug ts=2026-04-17T09:53:14.394057112Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB"
level=debug ts=2026-04-17T09:53:14.394079931Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.5 MB" series=8
level=info ts=2026-04-17T09:53:16.417088213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:53:16.429301003Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-17T09:53:16.611064179Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:53:16.61733523Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=warn ts=2026-04-17T09:53:16.629160714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:53:16.629152077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:53:16.629179555Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:53:16.629189877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:53:16.79630203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:53:18.427108539Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-17T09:53:18.915731961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:53:18.931314889Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:53:21.629095292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:53:22.450333304Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=debug ts=2026-04-17T09:53:26.480390518Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-17T09:53:26.629085808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:53:26.787699844Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:53:26.78776356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:53:26.795191422Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T09:53:30.510601793Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-17T09:53:31.629242756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:53:34.539501309Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=warn ts=2026-04-17T09:53:36.629254413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:53:37.100831365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T09:53:38.3483667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:53:38.358186955Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T09:53:38.569473417Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=warn ts=2026-04-17T09:53:41.629107332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:53:42.599694944Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-17T09:53:46.629171007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:53:46.629210349Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:53:46.629235491Z caller=checkpoint.go:390 msg="flushing series" totalSize="243 kB" series=7
level=debug ts=2026-04-17T09:53:46.629242833Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:53:46.62925524Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:53:46.633919301Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-17T09:53:46.635871316Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006961948s
level=warn ts=2026-04-17T09:53:51.629195566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:53:51.878903538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:53:51.889678578Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:53:56.629244972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:53:57.479944895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:39612"
level=debug ts=2026-04-17T09:53:58.263282051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:54:01.629088807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:54:02.96323753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:54:02.969823357Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-17T09:54:04.658409045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54610"
level=warn ts=2026-04-17T09:54:06.629351321Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:54:07.102413034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T09:54:07.3247811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:54:07.331346422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-17T09:54:10.363057584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:54:10.370898207Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-ohef\"}"
level=warn ts=2026-04-17T09:54:11.629119911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:54:13.457219467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:54:13.463944115Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:54:13.463974068Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:54:13.463999066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:54:13.469965276Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:54:13.788424157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:54:13.789979485Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-17T09:54:16.612898546Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:54:16.61311602Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:54:16.629962231Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:54:16.629998589Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:54:16.630028098Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:54:16.631064784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:54:16.631088585Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-17T09:54:16.631186278Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-04-17T09:54:16.631335447Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T09:54:20.661524772Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=warn ts=2026-04-17T09:54:21.629165491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:54:24.691532863Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-17T09:54:26.580582689Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:54:26.628553866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:54:27.48120691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:47420"
level=debug ts=2026-04-17T09:54:28.722206427Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=warn ts=2026-04-17T09:54:31.628927143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:54:32.128107976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:54:32.136813461Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:54:32.751594956Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-17T09:54:36.628753065Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:54:36.781192918Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
ts=2026-04-17T09:54:37.103901534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:54:40.813137989Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:54:41.628979428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:54:43.421719913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:39464"
level=debug ts=2026-04-17T09:54:44.841151619Z caller=checkpoint.go:378 msg="writing series" size="207 kB"
level=info ts=2026-04-17T09:54:46.629847123Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:54:46.629876966Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:54:46.629885358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:54:46.629906885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:54:48.87040742Z caller=checkpoint.go:378 msg="writing series" size="270 kB"
level=debug ts=2026-04-17T09:54:48.870430274Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-04-17T09:54:51.629116609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:54:51.836627902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:54:51.84428324Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:54:52.268212209Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:54:52.900452811Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-17T09:54:55.368462298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:54:55.374668962Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-17T09:54:56.503585632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:54:56.512296108Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:54:56.628748778Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:54:56.930476586Z caller=checkpoint.go:378 msg="writing series" size="732 kB"
ts=2026-04-17T09:54:57.482661981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:50980"
level=info ts=2026-04-17T09:54:59.20529673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:54:59.210104722Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:55:00.960821907Z caller=checkpoint.go:378 msg="writing series" size="373 kB"
level=debug ts=2026-04-17T09:55:00.960848238Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-04-17T09:55:01.628929353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:55:04.661853762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54856"
level=debug ts=2026-04-17T09:55:04.990176585Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-17T09:55:06.628810172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:55:06.804252678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:55:06.813792375Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-ohef\"}"
ts=2026-04-17T09:55:07.10543565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:55:09.019849448Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=warn ts=2026-04-17T09:55:11.628738216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:55:13.049635251Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=debug ts=2026-04-17T09:55:14.81748794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:55:16.609781986Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:55:16.610837339Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:55:16.629333884Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:55:16.629357703Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:55:16.629368381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:55:16.629377933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:55:17.080265946Z caller=checkpoint.go:378 msg="writing series" size="454 kB"
level=debug ts=2026-04-17T09:55:21.10976332Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-17T09:55:21.629059497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:55:25.139082719Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=warn ts=2026-04-17T09:55:26.629185742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:55:27.118107365Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-17T09:55:27.483865716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:44766"
level=debug ts=2026-04-17T09:55:29.168884092Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-17T09:55:31.628934233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:55:33.19928229Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=info ts=2026-04-17T09:55:33.800903345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:55:33.809899809Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:55:33.809918899Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:55:33.809943139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:55:33.816068889Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:55:36.628651341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:55:37.106764437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:55:37.229543398Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=debug ts=2026-04-17T09:55:37.229562063Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-04-17T09:55:37.751902942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:55:37.76003735Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:55:41.258673147Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=warn ts=2026-04-17T09:55:41.628778973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:55:43.166601865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:55:43.17102919Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:55:43.171051731Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:55:43.171079999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:55:43.174325567Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:55:45.288999174Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-17T09:55:46.628619208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:55:46.628637687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:55:46.628654894Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:55:46.628662141Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T09:55:49.085044124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.3 MB" avg_comp="2.4 MB" total_uncomp="63 MB" avg_uncomp="21 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:55:49.091542661Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:55:49.318277858Z caller=checkpoint.go:378 msg="writing series" size="212 kB"
level=warn ts=2026-04-17T09:55:51.629488958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:55:51.987181481Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:55:53.348857148Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=info ts=2026-04-17T09:55:53.458173804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:55:53.462251058Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:55:54.494970787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:55:54.498847276Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:55:56.628521115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:55:57.202722763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:55:57.209154348Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T09:55:57.378177384Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
ts=2026-04-17T09:55:57.485129599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:33436"
level=debug ts=2026-04-17T09:55:58.982927618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:56:01.40760092Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T09:56:01.628658617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:56:05.43787709Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-17T09:56:06.628535324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:56:07.107980954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:56:09.47712389Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-17T09:56:09.477146939Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=8
level=warn ts=2026-04-17T09:56:11.629509556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:56:13.497432886Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-17T09:56:14.074955816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:56:14.082334065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:56:16.351200698Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:56:16.609291376Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:56:16.610348289Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:56:16.629132834Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:56:16.62913647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:56:16.629150593Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:56:16.629158277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:56:17.527149652Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-04-17T09:56:21.557572092Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-17T09:56:21.628551916Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:56:24.744269097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:56:24.749881878Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:56:25.587073464Z caller=checkpoint.go:378 msg="writing series" size="259 kB"
level=warn ts=2026-04-17T09:56:26.628649293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:56:29.616722432Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=info ts=2026-04-17T09:56:30.344493353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:56:30.353565722Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:56:31.629054089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:56:33.647043992Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-17T09:56:33.664835549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:56:36.62867857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:56:37.109402958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:56:37.677144057Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-17T09:56:41.628958848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:56:41.706403565Z caller=checkpoint.go:378 msg="writing series" size="379 kB"
level=debug ts=2026-04-17T09:56:41.7064204Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=8
level=debug ts=2026-04-17T09:56:41.731426559Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:56:44.145990516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:56:44.151968865Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:56:45.736263536Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-04-17T09:56:46.629281407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:56:46.629289162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:56:46.62930355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:56:46.629312303Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:56:49.76621408Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-17T09:56:51.628785053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:56:53.796202248Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=warn ts=2026-04-17T09:56:56.629104426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:56:56.842936485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:56:56.852963717Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T09:56:57.826256103Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=info ts=2026-04-17T09:57:00.281553617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:57:00.288154869Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T09:57:01.628744605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:57:01.855447688Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-04-17T09:57:05.886079352Z caller=checkpoint.go:378 msg="writing series" size="723 kB"
level=debug ts=2026-04-17T09:57:05.886101849Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-04-17T09:57:06.629248683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:57:07.110521943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T09:57:09.915378017Z caller=checkpoint.go:378 msg="writing series" size="521 kB"
level=warn ts=2026-04-17T09:57:11.628595416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:57:12.891919732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:57:12.9004839Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:57:13.945394857Z caller=checkpoint.go:378 msg="writing series" size="612 kB"
level=debug ts=2026-04-17T09:57:13.945417417Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-04-17T09:57:15.626079827Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:57:16.173252275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:57:16.180861972Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:57:16.609657122Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:57:16.610719701Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:57:16.62934344Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:57:16.629363209Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:57:16.629372507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:57:16.629402954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:57:17.974715449Z caller=checkpoint.go:378 msg="writing series" size="643 B"
level=warn ts=2026-04-17T09:57:21.629470084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:57:22.005444206Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-17T09:57:22.757629902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T09:57:26.042969543Z caller=checkpoint.go:378 msg="writing series" size="15 MB"
level=debug ts=2026-04-17T09:57:26.042991964Z caller=checkpoint.go:390 msg="flushing series" totalSize="15 MB" series=3
level=warn ts=2026-04-17T09:57:26.629316633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:57:30.064511368Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=warn ts=2026-04-17T09:57:31.62865195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:57:34.094642304Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
ts=2026-04-17T09:57:34.670078454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:42680"
level=warn ts=2026-04-17T09:57:36.628731371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:57:36.727124988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:57:36.732340553Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-17T09:57:37.112319472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T09:57:38.124083626Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-17T09:57:38.478411561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:57:38.485993923Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T09:57:40.673290504Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:57:41.629556236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:57:42.154957418Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-17T09:57:42.154981262Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.3 MB" series=4
level=debug ts=2026-04-17T09:57:46.184177531Z caller=checkpoint.go:378 msg="writing series" size="3.4 kB"
level=info ts=2026-04-17T09:57:46.628504521Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:57:46.628535225Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:57:46.628546537Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:57:46.62855878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:57:48.109892914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:57:48.118929545Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:57:50.214032979Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-04-17T09:57:51.629442913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:57:54.244002481Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=warn ts=2026-04-17T09:57:56.62912393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:57:58.274521677Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-17T09:58:01.63084805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:58:02.303431846Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-17T09:58:06.333582749Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=debug ts=2026-04-17T09:58:06.53873849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T09:58:06.540363527Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T09:58:06.629378331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:58:07.114000685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:58:08.231438466Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T09:58:09.262298262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:58:09.269671017Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T09:58:10.36367087Z caller=checkpoint.go:378 msg="writing series" size="505 kB"
level=debug ts=2026-04-17T09:58:10.363700626Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-17T09:58:11.629153959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:58:12.5663557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:58:12.570565509Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T09:58:14.3951642Z caller=checkpoint.go:378 msg="writing series" size="5.1 MB"
level=debug ts=2026-04-17T09:58:14.395188033Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.1 MB" series=1
level=debug ts=2026-04-17T09:58:16.613914175Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:58:16.614042119Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T09:58:16.630659931Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:58:16.630695581Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:58:16.630718327Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:58:16.630913265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:58:18.425122181Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-17T09:58:21.629396133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:58:22.452826916Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=debug ts=2026-04-17T09:58:26.483081902Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-17T09:58:26.62915392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:58:27.491502434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:57096"
level=debug ts=2026-04-17T09:58:30.513056112Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-17T09:58:31.629138655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:58:31.839551456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:58:31.845272799Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:58:31.845293245Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T09:58:31.845315485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:58:31.848741968Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T09:58:32.573002934Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T09:58:34.542746163Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-04-17T09:58:35.508579616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:58:35.518047812Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
level=warn ts=2026-04-17T09:58:36.628737881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:58:37.115484704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:58:38.571757371Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=debug ts=2026-04-17T09:58:41.131865404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:58:41.629354553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:58:42.601645771Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-17T09:58:43.433469779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:57808"
level=info ts=2026-04-17T09:58:43.597063224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:58:43.603766013Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-17T09:58:46.628476123Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:58:46.628499255Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:58:46.628507653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:58:46.628518236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:58:46.631655973Z caller=checkpoint.go:390 msg="flushing series" totalSize="274 kB" series=7
level=info ts=2026-04-17T09:58:46.637446145Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-17T09:58:46.638740897Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007443494s
level=info ts=2026-04-17T09:58:47.040378002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:58:47.044269284Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:58:51.628749003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:58:53.963367747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:58:53.96779515Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T09:58:56.629182491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:59:01.132118511Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:59:01.62860263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:59:03.76170961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.5 MB" total_uncomp="107 MB" avg_uncomp="21 MB" full=4 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:59:03.768844145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:59:06.629493506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:59:07.116641837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=info ts=2026-04-17T09:59:07.668761249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T09:59:07.672700458Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T09:59:11.629319799Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T09:59:13.434918002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:34268"
level=info ts=2026-04-17T09:59:16.610066384Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T09:59:16.610107506Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T09:59:16.628524721Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-17T09:59:16.628524657Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T09:59:16.628552355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:59:16.62856071Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T09:59:16.62859071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:59:16.628653484Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-04-17T09:59:16.62889381Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T09:59:20.659518903Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=warn ts=2026-04-17T09:59:21.629143568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:59:22.527534269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T09:59:22.539078079Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T09:59:24.688953009Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-17T09:59:26.629301823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:59:28.719044406Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=debug ts=2026-04-17T09:59:31.25240286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T09:59:31.628703633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:59:32.748506718Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=debug ts=2026-04-17T09:59:34.412048317Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T09:59:36.629065992Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:59:36.778473978Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
ts=2026-04-17T09:59:37.118056157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T09:59:40.808354929Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T09:59:41.628715525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T09:59:41.659791708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T09:59:41.668390039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T09:59:44.838379733Z caller=checkpoint.go:378 msg="writing series" size="289 kB"
level=debug ts=2026-04-17T09:59:44.838403631Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-17T09:59:46.628487115Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T09:59:46.628519167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:59:46.628529717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T09:59:46.62854016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T09:59:48.868704119Z caller=checkpoint.go:378 msg="writing series" size="350 kB"
level=warn ts=2026-04-17T09:59:51.628658905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:59:52.897984898Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-17T09:59:52.994128015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T09:59:53.001543189Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T09:59:53.544040816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T09:59:53.551205195Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T09:59:56.628600019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T09:59:56.928455102Z caller=checkpoint.go:378 msg="writing series" size="644 kB"
level=debug ts=2026-04-17T10:00:00.958104345Z caller=checkpoint.go:378 msg="writing series" size="373 kB"
level=debug ts=2026-04-17T10:00:00.958129256Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-04-17T10:00:00.98311913Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:00:01.6295057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:00:03.535477819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:00:03.543722635Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:00:03.543744492Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:00:03.543769177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:00:03.550325687Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
ts=2026-04-17T10:00:04.677743403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:60982"
level=debug ts=2026-04-17T10:00:04.987961383Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=warn ts=2026-04-17T10:00:06.629028869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:00:06.717291787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:00:06.721997055Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-17T10:00:07.119701017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T10:00:09.017620131Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=warn ts=2026-04-17T10:00:11.62899542Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:00:13.047144121Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
level=debug ts=2026-04-17T10:00:16.609484416Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:00:16.610556329Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T10:00:16.610782023Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-17T10:00:16.610823912Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1973799 n=1
level=debug ts=2026-04-17T10:00:16.610865504Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1973799 n=1
level=info ts=2026-04-17T10:00:16.629406611Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T10:00:16.629410867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:00:16.629427369Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:00:16.629435319Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T10:00:17.077243867Z caller=checkpoint.go:378 msg="writing series" size="365 kB"
level=debug ts=2026-04-17T10:00:21.106553688Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-17T10:00:21.628587933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:00:22.062126357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:00:22.070893491Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:00:25.136388511Z caller=checkpoint.go:378 msg="writing series" size="5.8 kB"
level=debug ts=2026-04-17T10:00:25.331079321Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:00:26.628520621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:00:29.164932362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T10:00:29.166995686Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-17T10:00:31.6285846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:00:33.196756471Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=warn ts=2026-04-17T10:00:36.628554603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:00:37.121164434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T10:00:37.226503719Z caller=checkpoint.go:378 msg="writing series" size="382 kB"
level=debug ts=2026-04-17T10:00:37.226525348Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-04-17T10:00:37.73704638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:00:37.745613063Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-17T10:00:40.153195361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:00:40.159970303Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:00:40.159989291Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:00:40.160037407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:00:40.16598311Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:00:41.256236509Z caller=checkpoint.go:378 msg="writing series" size="281 kB"
level=warn ts=2026-04-17T10:00:41.629193138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:00:43.439537018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:40748"
level=debug ts=2026-04-17T10:00:45.286169417Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-17T10:00:46.628949869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:00:46.628940529Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:00:46.628970397Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:00:46.628980886Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T10:00:47.162579247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:00:47.166835636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T10:00:47.631473235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:00:47.635421822Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:00:49.315662544Z caller=checkpoint.go:378 msg="writing series" size="255 kB"
level=info ts=2026-04-17T10:00:51.359169732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:00:51.366647346Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
level=warn ts=2026-04-17T10:00:51.628939939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:00:52.739231025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:00:52.746343179Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:00:53.346111694Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=debug ts=2026-04-17T10:00:55.039837055Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:00:56.629128286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:00:57.376182746Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=debug ts=2026-04-17T10:01:01.405226896Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T10:01:01.629385427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:01:04.68091367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:44214"
level=debug ts=2026-04-17T10:01:05.435348969Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-17T10:01:06.628968409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:01:07.122165198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:01:09.477688126Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-17T10:01:09.477711838Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=8
level=warn ts=2026-04-17T10:01:11.629464299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:01:13.441219195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:52280"
level=debug ts=2026-04-17T10:01:13.494621638Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-17T10:01:16.610050862Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T10:01:16.610437516Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:01:16.628740275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:01:16.628759909Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:01:16.628769722Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:01:16.628790862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:01:17.524705513Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-04-17T10:01:19.260171999Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:01:20.115914125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T10:01:21.554814742Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=warn ts=2026-04-17T10:01:21.629103532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:01:22.185809118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="16 MB" avg_comp="2.6 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:01:22.19398818Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-17T10:01:22.891047845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:01:22.897466043Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T10:01:25.58477596Z caller=checkpoint.go:378 msg="writing series" size="372 kB"
level=warn ts=2026-04-17T10:01:26.629530887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:01:29.614806894Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=warn ts=2026-04-17T10:01:31.629224659Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:01:33.644579085Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=info ts=2026-04-17T10:01:34.116986013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:01:34.126767426Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
level=warn ts=2026-04-17T10:01:36.629392058Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:01:37.123279835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:01:37.674241417Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=debug ts=2026-04-17T10:01:37.674263046Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-17T10:01:41.628976003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:01:41.704453423Z caller=checkpoint.go:378 msg="writing series" size="479 kB"
level=info ts=2026-04-17T10:01:44.068294293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:01:44.074244882Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:01:45.734303895Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-04-17T10:01:46.629298942Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T10:01:46.629331409Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:01:46.62932812Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:01:46.629345258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T10:01:47.473389477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:01:47.478142141Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:01:49.763921588Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-17T10:01:50.872532052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:01:51.628792476Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:01:53.79357595Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=warn ts=2026-04-17T10:01:56.629239883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:01:57.500606246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:56862"
level=debug ts=2026-04-17T10:01:57.82408976Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=debug ts=2026-04-17T10:01:57.824122461Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-17T10:02:01.628747413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:02:01.853245161Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-04-17T10:02:04.557785342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T10:02:04.559929465Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-17T10:02:04.682828554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:33766"
level=debug ts=2026-04-17T10:02:05.884109502Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-17T10:02:05.884136446Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-04-17T10:02:06.188482103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:02:06.195706765Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:02:06.195728605Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:02:06.19575546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:02:06.20343577Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T10:02:06.629452785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:02:07.124377479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:02:09.913588632Z caller=checkpoint.go:378 msg="writing series" size="591 kB"
level=warn ts=2026-04-17T10:02:11.628987351Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:02:13.943448891Z caller=checkpoint.go:378 msg="writing series" size="768 kB"
level=debug ts=2026-04-17T10:02:13.943472827Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-04-17T10:02:16.610483828Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T10:02:16.615115711Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=warn ts=2026-04-17T10:02:16.631072391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:02:16.631112618Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:02:16.631125397Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:02:16.631135239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T10:02:16.694264853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="156 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:02:16.703550678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:02:17.972503459Z caller=checkpoint.go:378 msg="writing series" size="658 B"
level=info ts=2026-04-17T10:02:20.885248665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:02:20.8923203Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T10:02:21.630099662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:02:22.002487028Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-17T10:02:22.386974127Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:02:26.054772431Z caller=checkpoint.go:378 msg="writing series" size="20 MB"
level=debug ts=2026-04-17T10:02:26.054797233Z caller=checkpoint.go:390 msg="flushing series" totalSize="20 MB" series=3
level=warn ts=2026-04-17T10:02:26.630141996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:02:27.459446983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:02:27.467804785Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
ts=2026-04-17T10:02:27.501359998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:42704"
level=debug ts=2026-04-17T10:02:30.062253199Z caller=checkpoint.go:378 msg="writing series" size="204 kB"
level=warn ts=2026-04-17T10:02:31.630106074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:02:33.252240546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:02:33.257614546Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:02:33.257633808Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:02:33.257655279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:02:33.261024935Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:02:34.092467911Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=warn ts=2026-04-17T10:02:36.629127325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:02:37.125938841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T10:02:38.121699896Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-17T10:02:41.628534867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:02:42.1535761Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-17T10:02:42.153598996Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=4
ts=2026-04-17T10:02:43.446430319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:57150"
level=info ts=2026-04-17T10:02:44.787624511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:02:44.794259904Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T10:02:44.930809017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:02:44.935266278Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:02:46.182176733Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=debug ts=2026-04-17T10:02:46.606693233Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T10:02:46.629413567Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:02:46.629438601Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:02:46.629447264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:02:46.629465685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:02:50.211445706Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-04-17T10:02:51.629341131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:02:53.343858022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:02:53.352860979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T10:02:54.2416922Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=warn ts=2026-04-17T10:02:56.628518943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:02:58.270963773Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=debug ts=2026-04-17T10:03:01.06287354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T10:03:01.629516572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:03:02.30148367Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-17T10:03:06.330940384Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
level=warn ts=2026-04-17T10:03:06.629398469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:03:07.128341385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=info ts=2026-04-17T10:03:08.04314698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:03:08.053367468Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:03:10.360560218Z caller=checkpoint.go:378 msg="writing series" size="363 kB"
level=debug ts=2026-04-17T10:03:10.360588703Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-17T10:03:11.628596709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:03:13.142976852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:03:13.146988819Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:03:14.394160593Z caller=checkpoint.go:378 msg="writing series" size="8.1 MB"
level=debug ts=2026-04-17T10:03:14.394183403Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.1 MB" series=1
level=info ts=2026-04-17T10:03:16.289393871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="156 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:03:16.296960444Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-17T10:03:16.61088862Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T10:03:16.612043259Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=warn ts=2026-04-17T10:03:16.629256852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:03:16.629239078Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:03:16.629280323Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:03:16.629291131Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T10:03:18.423100729Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-17T10:03:21.630145775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:03:22.12916375Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:03:22.450448264Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=debug ts=2026-04-17T10:03:26.480371816Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-17T10:03:26.631079699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:03:30.510081349Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-17T10:03:31.629123851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:03:34.540299393Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=warn ts=2026-04-17T10:03:36.629144556Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:03:37.130457028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T10:03:37.615087231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:03:37.634732043Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:03:38.570259853Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-17T10:03:41.629093445Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:03:42.599806132Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-17T10:03:43.448861712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:45032"
level=debug ts=2026-04-17T10:03:43.997079825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T10:03:46.628925261Z caller=checkpoint.go:390 msg="flushing series" totalSize="324 kB" series=7
level=info ts=2026-04-17T10:03:46.629047123Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:03:46.629074074Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-17T10:03:46.629080574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:03:46.629084323Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T10:03:46.631872864Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-17T10:03:46.632761475Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.003917183s
level=info ts=2026-04-17T10:03:49.02616514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:03:49.03486383Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T10:03:51.629233577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:03:54.365846948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:03:54.369975059Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:03:54.900915511Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T10:03:56.608036408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:03:56.616104361Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-ohef\"}"
level=warn ts=2026-04-17T10:03:56.629209207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T10:04:01.629039545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T10:04:06.628582451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:04:07.132218662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=warn ts=2026-04-17T10:04:11.629143865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:04:11.892556859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="19 MB" avg_comp="2.7 MB" total_uncomp="156 MB" avg_uncomp="22 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:04:11.90283677Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:04:16.609190253Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:04:16.610262417Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T10:04:16.628441878Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-17T10:04:16.628479727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:04:16.628504413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:04:16.62851322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:04:16.628531576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:04:16.628575924Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=debug ts=2026-04-17T10:04:16.628750256Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T10:04:20.599809296Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=warn ts=2026-04-17T10:04:21.629202989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:04:22.226274573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:04:22.231907105Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:04:24.570242687Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-17T10:04:25.706826069Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:04:26.628612745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:04:27.508385861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:43458"
level=debug ts=2026-04-17T10:04:28.54132427Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=info ts=2026-04-17T10:04:29.185132158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:04:29.192453814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T10:04:31.629324911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:04:32.512089169Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
ts=2026-04-17T10:04:34.689552298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:54260"
level=debug ts=2026-04-17T10:04:36.482588599Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=warn ts=2026-04-17T10:04:36.629129688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:04:37.133651261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:04:40.453083782Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-17T10:04:41.176964189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T10:04:41.628821265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:04:44.4237065Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=debug ts=2026-04-17T10:04:44.423729172Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-17T10:04:44.638255169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:04:44.642003509Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T10:04:46.629332845Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:04:46.629357873Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:04:46.629366363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:04:46.629375823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:04:47.907362526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:04:47.91443245Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:04:48.393732201Z caller=checkpoint.go:378 msg="writing series" size="491 kB"
level=warn ts=2026-04-17T10:04:51.62849848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:04:52.364111316Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-17T10:04:55.377040951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:04:55.38406348Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:04:56.0869894Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:04:56.335949391Z caller=checkpoint.go:378 msg="writing series" size="804 kB"
level=debug ts=2026-04-17T10:04:56.335970307Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-04-17T10:04:56.62928131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:05:00.306237204Z caller=checkpoint.go:378 msg="writing series" size="373 kB"
level=warn ts=2026-04-17T10:05:01.629036134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:05:02.816364595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:05:02.82145177Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:05:02.821471512Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:05:02.821494512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:05:02.824503874Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:05:04.275900947Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-17T10:05:06.628820019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:05:07.139457247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T10:05:08.247279354Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=warn ts=2026-04-17T10:05:11.629551643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:05:12.217411292Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=info ts=2026-04-17T10:05:15.544543032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:05:15.555136909Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T10:05:16.188293103Z caller=checkpoint.go:378 msg="writing series" size="525 kB"
level=debug ts=2026-04-17T10:05:16.188317831Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-04-17T10:05:16.608982599Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:05:16.611112509Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T10:05:16.629125134Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:05:16.629147025Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:05:16.629154425Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:05:16.62914642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:05:19.918430289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:05:19.92247409Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:05:20.15926169Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-17T10:05:21.629521792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:05:24.129817639Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=warn ts=2026-04-17T10:05:26.629188945Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:05:27.510917315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:47472"
level=debug ts=2026-04-17T10:05:28.100094874Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-17T10:05:30.427183815Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:05:31.629328233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:05:31.682510713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:05:31.691093258Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:05:32.070911056Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=debug ts=2026-04-17T10:05:33.264913209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T10:05:36.041712334Z caller=checkpoint.go:378 msg="writing series" size="301 kB"
level=warn ts=2026-04-17T10:05:36.628603145Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:05:37.140964842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=info ts=2026-04-17T10:05:39.519251477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:05:39.522983076Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:05:40.011196357Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-04-17T10:05:41.629051129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:05:43.981886589Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=info ts=2026-04-17T10:05:46.030579015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:05:46.035145067Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-17T10:05:46.629042581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T10:05:46.629050265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:05:46.629070388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:05:46.629080034Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T10:05:47.953206879Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=info ts=2026-04-17T10:05:49.998965257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:05:50.005581635Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T10:05:51.629465654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:05:51.923812061Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=debug ts=2026-04-17T10:05:55.427911762Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:05:55.894208352Z caller=checkpoint.go:378 msg="writing series" size="77 kB"
level=warn ts=2026-04-17T10:05:56.62931413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:05:57.512722432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:58646"
level=debug ts=2026-04-17T10:05:59.86508758Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-17T10:06:01.629304188Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:03.835265924Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-17T10:06:06.628970456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:06:07.142183206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:06:07.813717709Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-17T10:06:07.813742379Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=13
level=warn ts=2026-04-17T10:06:11.629458601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:11.77615937Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=info ts=2026-04-17T10:06:13.455395493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:06:13.463022189Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:06:13.463040615Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:06:13.463066912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:06:13.47020258Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-ohef\"}"
level=info ts=2026-04-17T10:06:14.685556963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:06:14.692545425Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T10:06:15.746738317Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-04-17T10:06:16.609070078Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:06:16.6101412Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T10:06:16.629251943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T10:06:16.629259234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:16.629269821Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:06:16.629278089Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T10:06:19.717260285Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-04-17T10:06:21.554105897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:06:21.558206333Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-17T10:06:21.628833842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:23.688618095Z caller=checkpoint.go:378 msg="writing series" size="352 kB"
level=warn ts=2026-04-17T10:06:26.628763935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:27.15685668Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:06:27.659201317Z caller=checkpoint.go:378 msg="writing series" size="129 kB"
level=debug ts=2026-04-17T10:06:31.588718816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T10:06:31.59049158Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-17T10:06:31.628935697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:31.628960463Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=info ts=2026-04-17T10:06:32.988526467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:06:32.99670892Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-ohef\"}"
ts=2026-04-17T10:06:34.693576282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.85:45150"
level=debug ts=2026-04-17T10:06:35.599675986Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=debug ts=2026-04-17T10:06:35.599700813Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-17T10:06:36.628513076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:06:37.142861402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=info ts=2026-04-17T10:06:37.727427984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:06:37.731507515Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-17T10:06:37.892916123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:06:37.900435288Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:06:39.570958561Z caller=checkpoint.go:378 msg="writing series" size="373 kB"
level=warn ts=2026-04-17T10:06:41.629494595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:43.541623782Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=info ts=2026-04-17T10:06:46.62867797Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:06:46.628704255Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:06:46.628712197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:06:46.628746221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:47.511246921Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-17T10:06:51.481878186Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=warn ts=2026-04-17T10:06:51.62880043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:55.453371057Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=debug ts=2026-04-17T10:06:55.83747375Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:06:56.629250653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:06:59.423815845Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-17T10:07:01.628593216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:07:01.982602158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:07:01.987589988Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T10:07:02.875556712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:07:02.882624849Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:07:03.394419742Z caller=checkpoint.go:378 msg="writing series" size="973 kB"
level=debug ts=2026-04-17T10:07:03.394440732Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=warn ts=2026-04-17T10:07:06.629122417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:07:07.14372677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:07:07.364559342Z caller=checkpoint.go:378 msg="writing series" size="676 kB"
level=info ts=2026-04-17T10:07:09.754069841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:07:09.763040375Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T10:07:11.335567829Z caller=checkpoint.go:378 msg="writing series" size="860 kB"
level=debug ts=2026-04-17T10:07:11.33559097Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-04-17T10:07:11.629126412Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:07:15.305376081Z caller=checkpoint.go:378 msg="writing series" size="658 B"
level=debug ts=2026-04-17T10:07:16.609200783Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:07:16.610248142Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T10:07:16.628729675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:07:16.628750671Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:07:16.628760035Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:07:16.628774764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:07:19.276270129Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-17T10:07:21.628785661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:07:23.262435292Z caller=checkpoint.go:378 msg="writing series" size="24 MB"
level=debug ts=2026-04-17T10:07:23.262457239Z caller=checkpoint.go:390 msg="flushing series" totalSize="24 MB" series=3
level=debug ts=2026-04-17T10:07:24.130177615Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T10:07:24.871246118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:07:24.87986252Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:07:24.879882121Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:07:24.879911548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:07:24.886103291Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T10:07:26.62919999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:07:27.189984336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T10:07:27.217832382Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=debug ts=2026-04-17T10:07:31.188015248Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=warn ts=2026-04-17T10:07:31.628551027Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:07:35.158243811Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-17T10:07:35.613080105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:07:35.617943633Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T10:07:36.629231737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:07:37.144510394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=info ts=2026-04-17T10:07:38.208949401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:07:38.216427032Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:07:39.132030452Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-17T10:07:39.132054762Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=4
level=warn ts=2026-04-17T10:07:41.628984753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:07:43.099598805Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-04-17T10:07:46.629325985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:07:46.629362483Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-17T10:07:46.629365141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:07:46.629384792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T10:07:46.963031355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:07:46.967876928Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:07:47.070942528Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=info ts=2026-04-17T10:07:48.116381661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:07:48.125733836Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:07:51.041120789Z caller=checkpoint.go:378 msg="writing series" size="342 kB"
level=warn ts=2026-04-17T10:07:51.629618994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:07:51.776908319Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:07:55.011752295Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=warn ts=2026-04-17T10:07:56.629379703Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:07:57.517742184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.92:59596"
level=debug ts=2026-04-17T10:07:58.982470369Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-17T10:07:59.235074243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:07:59.245147765Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T10:08:01.629959044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:08:02.952365311Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=warn ts=2026-04-17T10:08:06.628920327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:08:06.925735975Z caller=checkpoint.go:378 msg="writing series" size="645 kB"
level=debug ts=2026-04-17T10:08:06.925767058Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
ts=2026-04-17T10:08:07.146381296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T10:08:08.899735464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-17T10:08:10.902294133Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-17T10:08:10.902314685Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=1
level=warn ts=2026-04-17T10:08:11.629197527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:08:13.46081395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:35160"
level=debug ts=2026-04-17T10:08:14.865041663Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-17T10:08:16.149378152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:08:16.157425134Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-17T10:08:16.612062389Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T10:08:16.612073263Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=warn ts=2026-04-17T10:08:16.632101928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:08:16.632148074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:08:16.632160954Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:08:16.632169036Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T10:08:18.835061503Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=debug ts=2026-04-17T10:08:19.058759866Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:08:21.629130388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:08:22.707895089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:08:22.73032191Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:08:22.80809935Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-17T10:08:26.629074613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:08:26.77608011Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-04-17T10:08:30.338392304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:08:30.343209956Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:08:30.343228283Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:08:30.343249705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:08:30.346571929Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:08:30.747405662Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=warn ts=2026-04-17T10:08:31.629098453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:08:34.719128789Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-17T10:08:36.629218844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:08:37.14751602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T10:08:38.688251991Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=warn ts=2026-04-17T10:08:41.629085301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:08:42.658448734Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=info ts=2026-04-17T10:08:44.965442168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:08:44.976459078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:08:44.9764844Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:08:44.976518616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:08:44.98348459Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-17T10:08:46.072136621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:08:46.075867877Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T10:08:46.282863562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:08:46.291168304Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:08:46.629404241Z caller=checkpoint.go:390 msg="flushing series" totalSize="595 kB" series=8
level=warn ts=2026-04-17T10:08:46.629555958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:08:46.629595318Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:08:46.62960899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:08:46.629617745Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T10:08:46.632965149Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-17T10:08:46.634358852Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005648428s
level=debug ts=2026-04-17T10:08:48.594241144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T10:08:51.629488257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:08:53.959527012Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T10:08:55.379134251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:08:55.386373706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-17T10:08:56.62910353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-17T10:09:01.629142926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:09:02.542976593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:09:02.551156984Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-17T10:09:06.629270717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:09:07.14923645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=warn ts=2026-04-17T10:09:11.629092574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:09:15.101388072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:09:15.115445361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-17T10:09:16.610102086Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-17T10:09:16.611097631Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:09:16.629078861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:09:16.629125649Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-17T10:09:16.629092304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:09:16.629137457Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-17T10:09:16.629150003Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-17T10:09:16.629272719Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=debug ts=2026-04-17T10:09:16.629509977Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-17T10:09:20.6013283Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=warn ts=2026-04-17T10:09:21.629227336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:09:22.611079172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:09:22.629718621Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:09:24.079594097Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:09:24.571446687Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-17T10:09:26.629298105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:09:28.541493951Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=warn ts=2026-04-17T10:09:31.629290049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:09:31.854567012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:09:31.86079174Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T10:09:32.513113466Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-17T10:09:34.193077021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:09:34.200087949Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:09:36.483309334Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=warn ts=2026-04-17T10:09:36.63116895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:09:37.151034046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:09:40.453675994Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-17T10:09:41.629108028Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:09:44.425032165Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=debug ts=2026-04-17T10:09:45.834211849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-17T10:09:46.629203865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:09:46.629239536Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:09:46.629250193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:09:46.62929985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:09:46.630506404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:09:46.63902667Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:09:48.395320639Z caller=checkpoint.go:378 msg="writing series" size="353 kB"
level=debug ts=2026-04-17T10:09:48.395344346Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=warn ts=2026-04-17T10:09:51.63012934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:09:52.365329669Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-17T10:09:55.841964736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:09:56.336790489Z caller=checkpoint.go:378 msg="writing series" size="909 kB"
level=warn ts=2026-04-17T10:09:56.629328584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:00.307260381Z caller=checkpoint.go:378 msg="writing series" size="373 kB"
level=debug ts=2026-04-17T10:10:00.307292145Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-04-17T10:10:01.629141293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:04.277322261Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
level=warn ts=2026-04-17T10:10:06.629462576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:10:07.151924985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T10:10:08.247373183Z caller=checkpoint.go:378 msg="writing series" size="274 kB"
level=warn ts=2026-04-17T10:10:11.629217821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:12.218153925Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=info ts=2026-04-17T10:10:12.321094074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:10:12.332875408Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:10:16.189319286Z caller=checkpoint.go:378 msg="writing series" size="392 kB"
level=debug ts=2026-04-17T10:10:16.609700835Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:10:16.61074628Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T10:10:16.629032481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:10:16.629061991Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:10:16.629070631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:10:16.62907329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:10:17.146630961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:10:17.152260105Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T10:10:19.176204208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:10:19.182693529Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-17T10:10:20.159587074Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-17T10:10:21.629150218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:22.866556913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-17T10:10:22.868366687Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-17T10:10:22.922657063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:10:22.931139275Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:10:24.13046652Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=warn ts=2026-04-17T10:10:26.628845079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:28.100202533Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-17T10:10:28.169423234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:10:31.628566002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:32.070757479Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=info ts=2026-04-17T10:10:34.233282067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.7 MB" total_uncomp="180 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:10:34.243141347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:10:36.042234581Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=debug ts=2026-04-17T10:10:36.04225654Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=warn ts=2026-04-17T10:10:36.629077327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:10:37.153745917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T10:10:40.012542473Z caller=checkpoint.go:378 msg="writing series" size="149 kB"
level=warn ts=2026-04-17T10:10:41.629533976Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:43.98330233Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-17T10:10:46.62911816Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T10:10:46.629132244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:46.629148657Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:10:46.629156798Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T10:10:47.953309816Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=warn ts=2026-04-17T10:10:51.628842718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:51.924300526Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=debug ts=2026-04-17T10:10:54.953684823Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-17T10:10:55.89442241Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-17T10:10:56.628731781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:10:59.864969003Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-17T10:11:00.039365766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-17T10:11:01.628664986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:11:03.222913863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.9 MB" avg_comp="2.5 MB" total_uncomp="85 MB" avg_uncomp="21 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:11:03.232397167Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-17T10:11:03.388664888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:11:03.392638982Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-17T10:11:03.835612562Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-17T10:11:06.628814091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:11:07.155132515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-2998cca2 10.129.0.92:7946"
level=debug ts=2026-04-17T10:11:07.817441517Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-17T10:11:07.817464658Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=8
level=info ts=2026-04-17T10:11:10.81153069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:11:10.815726915Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-17T10:11:11.628527047Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:11:11.776928264Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
ts=2026-04-17T10:11:13.468701872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:44996"
level=debug ts=2026-04-17T10:11:15.747512044Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=debug ts=2026-04-17T10:11:16.609405706Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.73:11211 count=1
level=info ts=2026-04-17T10:11:16.610443793Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-17T10:11:16.62854952Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-17T10:11:16.628562425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:11:16.628569149Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:11:16.628579565Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-17T10:11:19.718727279Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-04-17T10:11:19.982387503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:11:19.989293419Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:11:20.523203802Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-17T10:11:21.629078005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:11:22.992613297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="204 MB" avg_uncomp="23 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:11:23.001044019Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:11:23.68910289Z caller=checkpoint.go:378 msg="writing series" size="413 kB"
level=warn ts=2026-04-17T10:11:26.629048837Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:11:27.659642466Z caller=checkpoint.go:378 msg="writing series" size="136 kB"
level=warn ts=2026-04-17T10:11:31.628920998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:11:31.630038472Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=debug ts=2026-04-17T10:11:35.600297912Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=debug ts=2026-04-17T10:11:35.600325004Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-04-17T10:11:36.62937386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:11:37.008769962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-17T10:11:37.156308846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-g2pl2-ff8e6a07 10.129.0.87:7946"
level=debug ts=2026-04-17T10:11:39.57166544Z caller=checkpoint.go:378 msg="writing series" size="475 kB"
level=warn ts=2026-04-17T10:11:41.629112559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-17T10:11:43.470393639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.87:56190"
level=debug ts=2026-04-17T10:11:43.542324219Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=debug ts=2026-04-17T10:11:44.693959624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-17T10:11:45.382863945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="444 kB" avg_comp="444 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-17T10:11:45.386849572Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19d9ab83734%3A19d9ab83ac3%3Af75f0a74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-17T10:11:46.629104565Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-17T10:11:46.629129612Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-17T10:11:46.629140304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-17T10:11:46.629162244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:11:47.512697966Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-17T10:11:51.483068672Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=warn ts=2026-04-17T10:11:51.629267307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:11:54.822548486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="102 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-17T10:11:54.830910839Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19d9ab81c65%3A19d9ab91aa9%3Adaf05516?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-17T10:11:55.429429907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:11:55.433325989Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:11:55.433344483Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-17T10:11:55.433370081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:11:55.437229585Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-17T10:11:55.453405998Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=debug ts=2026-04-17T10:11:55.453418981Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-17T10:11:56.628787022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:11:59.423899221Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-17T10:12:01.628669304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-17T10:12:03.395845622Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-17T10:12:03.395868184Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-04-17T10:12:06.629500715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:12:07.049969318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="310 kB" avg_comp="310 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-17T10:12:07.056671695Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19d9ab9177c%3A19d9ab99c91%3A83fe965e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-17T10:12:07.157090542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-hfsrb-6f98f524 10.129.0.85:7946"
level=debug ts=2026-04-17T10:12:07.365792479Z caller=checkpoint.go:378 msg="writing series" size="812 kB"
level=info ts=2026-04-17T10:12:09.044145042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7e221b3e81fb6e1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-ohef\"}"
level=error ts=2026-04-17T10:12:09.051529052Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7e221b3e81fb6e1d/19d9acde7e0%3A19d9adb9aae%3Ae1c7b482?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-ohef\"}"
level=debug ts=2026-04-17T10:12:11.33641769Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-17T10:12:11.336440927Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=2
level=warn ts=2026-04-17T10:12:11.629363794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-17T10:12:12.602654634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="204 MB" avg_uncomp="23 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-17T10:12:12.609236618Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19d9ab83afa%3A19d9ad0592e%3Aeeb058d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-17T10:12:13.22761984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"