level=info ts=2026-04-22T13:33:18.709749917Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T13:33:18.709778708Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T13:33:18.711577166Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T13:33:18.711970037Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T13:33:18.712051182Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.78 inf=eth0
level=debug ts=2026-04-22T13:33:18.712590966Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T13:33:18.712665954Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.78 inf=eth0
level=info ts=2026-04-22T13:33:18.712711389Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-d08480c9
ts=2026-04-22T13:33:18.712788076Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T13:33:18.712800631Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.78 advertisePort=7946
level=debug ts=2026-04-22T13:33:18.712818519Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.78 advertisePort=7946
level=info ts=2026-04-22T13:33:18.712919765Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T13:33:18.719369263Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T13:33:18.719387637Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T13:33:18.719397239Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.479486ms
level=error ts=2026-04-22T13:33:18.719404117Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T13:33:18.719415035Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T13:33:18.721810616Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:33:18.721825372Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T13:33:18.728912998Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T13:33:18.728934622Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T13:33:18.728957297Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-22T13:33:18.728967313Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-22T13:33:18.728998469Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-22T13:33:18.728992879Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-22T13:33:18.729049665Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T13:33:18.729076649Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-22T13:33:18.72916416Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-22T13:33:18.729172874Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T13:33:18.729208958Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T13:33:18.729224638Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T13:33:18.729251342Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T13:33:18.729388204Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T13:33:18.841676755Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T13:33:18.841734622Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T13:33:19.842393001Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-22T13:33:19.998803944Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T13:33:19.998857175Z caller=loki.go:599 msg="Loki started" startup_time=1.34891113s
ts=2026-04-22T13:33:20.074863085Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:33:20.074883594Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:33:22.726907181Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:33:22.726930737Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T13:33:24.998903416Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T13:33:24.998956479Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T13:33:28.83885942Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T13:33:28.838876794Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T13:33:31.107105751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:33:39.383945016Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.69:7946: dial tcp 10.129.0.69:7946: connect: connection refused"
ts=2026-04-22T13:33:39.384823311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.70:7946"
ts=2026-04-22T13:33:39.386836059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.74:7946"
level=info ts=2026-04-22T13:33:39.387575124Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=20.668155796s
level=debug ts=2026-04-22T13:33:45.054925905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:33:55.407982235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T13:34:07.458593631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:34:25.409950611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:34:27.555879956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45252"
level=debug ts=2026-04-22T13:34:27.818572829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:34:38.646363794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60066"
ts=2026-04-22T13:34:55.411562233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T13:35:01.769958033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T13:35:01.771646621Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:35:13.431455735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:60454"
ts=2026-04-22T13:35:25.413028325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:35:38.650151706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38930"
level=debug ts=2026-04-22T13:35:39.15752707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:35:55.414932549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:36:08.652588003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33506"
ts=2026-04-22T13:36:25.416933304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
ts=2026-04-22T13:36:38.654051716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44046"
level=debug ts=2026-04-22T13:36:52.908779901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:36:55.418597474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:36:57.563486592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49384"
ts=2026-04-22T13:37:13.437776938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:56164"
ts=2026-04-22T13:37:25.420598088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:37:43.440266673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59266"
ts=2026-04-22T13:37:55.422114459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
level=debug ts=2026-04-22T13:37:56.29940144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:37:57.566420242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43738"
ts=2026-04-22T13:38:13.441925214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:37452"
ts=2026-04-22T13:38:25.423964977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:38:27.568499835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40232"
level=debug ts=2026-04-22T13:38:49.612835186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T13:38:55.425777843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
ts=2026-04-22T13:39:08.660642077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55004"
ts=2026-04-22T13:39:25.427870691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:39:43.446327955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:55310"
level=debug ts=2026-04-22T13:39:51.049786592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T13:39:51.051573013Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:39:55.429495442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:39:57.573689583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:48704"
ts=2026-04-22T13:40:25.431621669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
level=debug ts=2026-04-22T13:40:39.977539996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:40:55.434068065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:41:08.666177583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39072"
level=debug ts=2026-04-22T13:41:16.450718298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:41:25.436428509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
ts=2026-04-22T13:41:38.668113976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39906"
ts=2026-04-22T13:41:55.438075896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:41:57.579247195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56272"
level=debug ts=2026-04-22T13:41:59.524803666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:42:13.454112812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:36650"
ts=2026-04-22T13:42:25.440430063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T13:42:39.57657809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:42:55.442826034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:42:57.582270598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:47650"
level=debug ts=2026-04-22T13:43:15.544329019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T13:43:15.545855335Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T13:43:25.001257844Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T13:43:25.005472196Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:43:25.007942856Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:43:25.00966157Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:43:25.009691154Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T13:43:25.009704097Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-22T13:43:25.444547918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:43:27.58438937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35166"
ts=2026-04-22T13:43:38.673020944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39604"
level=debug ts=2026-04-22T13:43:53.156603852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:43:55.446077113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
ts=2026-04-22T13:44:08.675391636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54044"
ts=2026-04-22T13:44:25.447834298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T13:44:37.985503279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:44:43.461726078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44048"
ts=2026-04-22T13:44:55.449248432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
ts=2026-04-22T13:45:08.678243493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59964"
ts=2026-04-22T13:45:25.451498507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:45:27.590189271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36844"
level=debug ts=2026-04-22T13:45:39.048723861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:45:43.4652921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:43204"
ts=2026-04-22T13:45:55.453120376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:45:57.591839459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34798"
level=debug ts=2026-04-22T13:46:23.885645495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:46:25.455255282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:46:27.59334659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:46156"
ts=2026-04-22T13:46:38.682659727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:56714"
ts=2026-04-22T13:46:43.468488093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48930"
ts=2026-04-22T13:46:55.457634222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:47:13.470662336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:39416"
level=debug ts=2026-04-22T13:47:14.791555342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T13:47:14.793604208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:47:25.459926722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:47:55.461773888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T13:47:57.435580638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:48:13.473830679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:46900"
ts=2026-04-22T13:48:25.463462053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T13:48:53.376212038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:48:55.465294977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:48:57.601646976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44112"
ts=2026-04-22T13:49:25.467295791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:49:27.604129513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43912"
ts=2026-04-22T13:49:38.690724116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43678"
level=debug ts=2026-04-22T13:49:53.457440983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:49:55.469189281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:49:57.606159547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51174"
ts=2026-04-22T13:50:25.471732042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T13:50:47.351905509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:50:55.47385448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
ts=2026-04-22T13:50:57.609577408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58572"
ts=2026-04-22T13:51:08.694556674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52614"
ts=2026-04-22T13:51:25.475789494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T13:51:33.437534949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T13:51:33.439800699Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:51:55.478187554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T13:52:11.113229324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:52:25.480172479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:52:38.703090145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37528"
level=debug ts=2026-04-22T13:52:48.298229697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:52:55.482682037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=error ts=2026-04-22T13:53:25.014079205Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:53:25.016068654Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:53:25.017911882Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T13:53:25.017938008Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:53:25.4851285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
ts=2026-04-22T13:53:43.489061175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42646"
level=debug ts=2026-04-22T13:53:48.761390052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:53:55.487008731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:54:25.488636685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
level=debug ts=2026-04-22T13:54:35.431003843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:54:38.708796547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34930"
ts=2026-04-22T13:54:55.49097411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:54:57.62834499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53618"
level=debug ts=2026-04-22T13:55:17.663939024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T13:55:17.665804393Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:55:25.493006551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:55:43.495636728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:58102"
ts=2026-04-22T13:55:55.49451459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:55:57.632087696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:46162"
level=debug ts=2026-04-22T13:56:08.038439162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:56:08.719164331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48676"
ts=2026-04-22T13:56:25.497065608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T13:56:50.175591486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:56:55.498789744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:57:08.724068263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37184"
ts=2026-04-22T13:57:25.502973823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:57:27.64133227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35642"
level=debug ts=2026-04-22T13:57:30.574977553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:57:55.504848808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:58:25.507439489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
level=debug ts=2026-04-22T13:58:32.374997401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T13:58:55.509322114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T13:59:08.730508385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49476"
ts=2026-04-22T13:59:13.509499313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42928"
level=debug ts=2026-04-22T13:59:13.739734257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T13:59:13.741670926Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T13:59:25.511521336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:59:27.650421173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53244"
ts=2026-04-22T13:59:38.732870996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41986"
ts=2026-04-22T13:59:55.513861177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T13:59:57.652351659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:55244"
level=debug ts=2026-04-22T14:00:14.059850322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:00:25.515996735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:00:27.654837683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:46606"
ts=2026-04-22T14:00:55.517779026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:01:04.262483228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:01:13.515274113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:44124"
ts=2026-04-22T14:01:25.520153184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:01:27.658810664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49858"
level=debug ts=2026-04-22T14:01:45.613056091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:01:55.530280859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8c6af7f8 10.129.0.72:7946"
ts=2026-04-22T14:02:03.471850003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59334"
ts=2026-04-22T14:02:03.478252957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:59340"
ts=2026-04-22T14:02:25.539306091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-25ef732b 10.129.0.72:7946"
ts=2026-04-22T14:02:30.714050842Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8c6af7f8 (timeout reached)"
ts=2026-04-22T14:02:33.476256851Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-8c6af7f8 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T14:02:33.713961764Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8c6af7f8 has failed, no acks received"
ts=2026-04-22T14:02:43.520078721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59788"
level=debug ts=2026-04-22T14:02:47.840414128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:02:55.546268896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-25ef732b 10.129.0.72:7946"
ts=2026-04-22T14:02:57.663346157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60702"
level=error ts=2026-04-22T14:03:25.015421478Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:03:25.017424393Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:03:25.019187734Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:03:25.019222546Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:03:25.552302936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-25ef732b 10.129.0.72:7946"
level=debug ts=2026-04-22T14:03:46.869187346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:03:46.871183434Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:03:51.958822719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49020"
ts=2026-04-22T14:03:55.554834738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-25ef732b 10.129.0.72:7946"
ts=2026-04-22T14:04:21.960367181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37774"
ts=2026-04-22T14:04:25.557372737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-25ef732b 10.129.0.72:7946"
level=debug ts=2026-04-22T14:04:39.117955598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:04:55.559698746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-25ef732b 10.129.0.72:7946"
ts=2026-04-22T14:05:25.561707303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:05:29.599186972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:05:51.96521244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34308"
ts=2026-04-22T14:05:55.564116935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-25ef732b 10.129.0.72:7946"
ts=2026-04-22T14:05:57.671955622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34914"
level=debug ts=2026-04-22T14:06:11.907682058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:06:21.967560646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36660"
ts=2026-04-22T14:06:25.565722009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:06:51.969707584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50328"
level=debug ts=2026-04-22T14:06:52.692982629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:06:55.568100407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-25ef732b 10.129.0.72:7946"
ts=2026-04-22T14:07:21.971612204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55810"
ts=2026-04-22T14:07:25.570770434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:07:51.244339648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:07:51.246257433Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:07:55.572679055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:08:15.713901209Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-25ef732b (timeout reached)"
ts=2026-04-22T14:08:18.712990087Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-25ef732b has failed, no acks received"
ts=2026-04-22T14:08:20.713201024Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-25ef732b (timeout reached)"
ts=2026-04-22T14:08:23.7140321Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-25ef732b has failed, no acks received"
ts=2026-04-22T14:08:23.800932694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48320"
ts=2026-04-22T14:08:23.807896135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48330"
ts=2026-04-22T14:08:25.574611401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:08:43.537743234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:57192"
level=debug ts=2026-04-22T14:08:43.537839976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:08:55.57710636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:09:18.415827816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:09:25.578716139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:09:27.682570011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36870"
ts=2026-04-22T14:09:48.444263903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55348"
ts=2026-04-22T14:09:55.580731724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:09:57.68459107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44394"
level=debug ts=2026-04-22T14:10:06.473941016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:10:25.582972592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:10:48.447654921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55304"
level=debug ts=2026-04-22T14:10:55.023675489Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:10:55.584705298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d3eb3c96 10.129.0.72:7946"
ts=2026-04-22T14:10:57.688072667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44058"
ts=2026-04-22T14:11:25.586565144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:11:48.451649299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41516"
ts=2026-04-22T14:11:55.588412375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d3eb3c96 10.129.0.72:7946"
level=debug ts=2026-04-22T14:11:56.598420024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:11:56.600377641Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:12:13.548901723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:53832"
ts=2026-04-22T14:12:18.453286739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60182"
ts=2026-04-22T14:12:25.590666711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:12:27.693085888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51142"
level=debug ts=2026-04-22T14:12:40.696479152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:12:43.551210619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:36540"
ts=2026-04-22T14:12:55.592306585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=error ts=2026-04-22T14:13:25.012890417Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:13:25.016919346Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:13:25.019892414Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:13:25.019930259Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:13:25.593970166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:13:27.053863072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:13:27.696592134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49370"
ts=2026-04-22T14:13:43.554719448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:43896"
ts=2026-04-22T14:13:55.596168369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:13:57.698498952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58886"
level=debug ts=2026-04-22T14:14:06.056446824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:14:18.460259273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35922"
ts=2026-04-22T14:14:25.598698517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:14:48.466397446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51450"
ts=2026-04-22T14:14:55.600726269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:14:57.428121534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:14:57.702271628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49292"
ts=2026-04-22T14:15:13.560442406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:39414"
ts=2026-04-22T14:15:18.468920731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39826"
ts=2026-04-22T14:15:25.60229981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:15:27.704056388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57826"
level=debug ts=2026-04-22T14:15:38.060421305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:15:38.062786458Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:15:40.7146223Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d3eb3c96 (timeout reached)"
ts=2026-04-22T14:15:43.562273221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:46736"
ts=2026-04-22T14:15:43.713975635Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d3eb3c96 has failed, no acks received"
ts=2026-04-22T14:15:50.714449571Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d3eb3c96 (timeout reached)"
ts=2026-04-22T14:15:53.713987664Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d3eb3c96 has failed, no acks received"
ts=2026-04-22T14:15:55.604563223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:16:02.75463966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53210"
ts=2026-04-22T14:16:02.761708314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53214"
ts=2026-04-22T14:16:13.563902163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48146"
level=debug ts=2026-04-22T14:16:20.850269333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:16:25.606947473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:16:27.707918395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37998"
ts=2026-04-22T14:16:43.566266108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:40086"
ts=2026-04-22T14:16:52.712338565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37452"
ts=2026-04-22T14:16:55.608460778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:16:57.709342603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49928"
level=debug ts=2026-04-22T14:17:00.018057267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:17:13.568587056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:51658"
ts=2026-04-22T14:17:25.610307534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5a871f31 10.129.0.72:7946"
ts=2026-04-22T14:17:43.570398136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:32944"
level=debug ts=2026-04-22T14:17:48.398748135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:17:52.716545458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48966"
ts=2026-04-22T14:17:55.612038945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:17:57.71235564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60130"
ts=2026-04-22T14:18:25.613636366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:18:27.714476988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43560"
level=debug ts=2026-04-22T14:18:36.191818257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:18:55.615184962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:19:22.721860528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:58168"
ts=2026-04-22T14:19:25.61764237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:19:27.718006855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37020"
level=debug ts=2026-04-22T14:19:33.217815394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:19:33.219692025Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:19:55.619714996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5a871f31 10.129.0.72:7946"
ts=2026-04-22T14:20:13.577555505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:33594"
level=debug ts=2026-04-22T14:20:24.833455462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:20:25.621352586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:20:55.623175644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:21:03.889226519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:21:13.581009296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:40702"
ts=2026-04-22T14:21:22.727562423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35252"
ts=2026-04-22T14:21:25.62520839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:21:27.723614501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51086"
level=debug ts=2026-04-22T14:21:41.657630295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:21:55.627327562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5a871f31 10.129.0.72:7946"
level=debug ts=2026-04-22T14:22:20.164853684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:22:25.629730101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:22:55.632206841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:22:57.727183598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:44082"
level=debug ts=2026-04-22T14:23:19.214405888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:23:19.216255039Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:23:22.733646658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52554"
level=error ts=2026-04-22T14:23:25.012782361Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:23:25.014449844Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:23:25.016118992Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:23:25.016172899Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:23:25.634287644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5a871f31 10.129.0.72:7946"
ts=2026-04-22T14:23:27.7287723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40518"
ts=2026-04-22T14:23:43.587968629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:50230"
ts=2026-04-22T14:23:55.636377701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5a871f31 10.129.0.72:7946"
level=debug ts=2026-04-22T14:24:16.702283609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:24:25.638321805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:24:25.713989643Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-5a871f31 (timeout reached)"
ts=2026-04-22T14:24:27.73199525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34066"
ts=2026-04-22T14:24:28.713676925Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-5a871f31 has failed, no acks received"
ts=2026-04-22T14:24:38.675402403Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-5a871f31 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-22T14:24:53.422566745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:24:55.639865124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:25:00.739682773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50142"
ts=2026-04-22T14:25:00.745948538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50148"
ts=2026-04-22T14:25:13.592834825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:34386"
ts=2026-04-22T14:25:25.641586552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:25:27.736100865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51084"
ts=2026-04-22T14:25:45.082104989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55954"
level=debug ts=2026-04-22T14:25:48.801362386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:25:55.643855471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T14:26:22.753200544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:26:25.645477659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:26:55.647629583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd2566bf 10.129.0.72:7946"
level=debug ts=2026-04-22T14:27:13.333643526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:27:13.335204428Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:27:13.600612744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:47830"
ts=2026-04-22T14:27:15.086508437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55492"
ts=2026-04-22T14:27:25.649918562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:27:27.742663527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58634"
ts=2026-04-22T14:27:55.652148823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:27:57.745195321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42178"
level=debug ts=2026-04-22T14:28:10.2061782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:28:15.090734536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35298"
ts=2026-04-22T14:28:25.654245983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd2566bf 10.129.0.72:7946"
ts=2026-04-22T14:28:27.747126845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39216"
ts=2026-04-22T14:28:43.606048757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:40270"
ts=2026-04-22T14:28:55.656204039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cd2566bf 10.129.0.72:7946"
level=debug ts=2026-04-22T14:29:04.847154383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:29:25.657945448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:29:55.659530572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:29:56.859708749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:30:13.610966207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59770"
ts=2026-04-22T14:30:25.661921561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:30:40.052263635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:30:43.613072181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:57804"
ts=2026-04-22T14:30:55.664090527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:30:57.754746041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56716"
ts=2026-04-22T14:31:13.615101688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:57980"
ts=2026-04-22T14:31:15.098099076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54408"
ts=2026-04-22T14:31:25.665900587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:31:27.057115914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:31:27.058735047Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:31:55.668277311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:31:57.757987665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:52572"
level=debug ts=2026-04-22T14:32:21.061737039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:32:25.67022377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:32:43.619519157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:40328"
ts=2026-04-22T14:32:45.10350779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:57360"
ts=2026-04-22T14:32:55.671786259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T14:33:04.565434298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:33:25.014609681Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:33:25.016534013Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:33:25.018440918Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:33:25.01846596Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:33:25.673830629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T14:33:48.594046724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:33:55.675356136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:33:55.713363157Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cd2566bf (timeout reached)"
ts=2026-04-22T14:33:58.713696635Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cd2566bf has failed, no acks received"
ts=2026-04-22T14:34:13.624821457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:39948"
ts=2026-04-22T14:34:18.714170583Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-cd2566bf as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T14:34:20.714564263Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cd2566bf (timeout reached)"
ts=2026-04-22T14:34:23.713914047Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cd2566bf has failed, no acks received"
level=debug ts=2026-04-22T14:34:25.030316068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:34:25.677679543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:34:27.763995721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:56138"
ts=2026-04-22T14:34:43.626976154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38010"
ts=2026-04-22T14:34:55.679483978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:34:57.765423703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45216"
level=debug ts=2026-04-22T14:35:07.203560632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:35:07.205355999Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:35:12.721150599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48246"
ts=2026-04-22T14:35:12.729170796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48250"
ts=2026-04-22T14:35:13.629866502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:54604"
ts=2026-04-22T14:35:25.681705249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:35:43.632276697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:54436"
ts=2026-04-22T14:35:55.683491418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f92f37df 10.129.0.72:7946"
ts=2026-04-22T14:35:57.768728256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:55642"
ts=2026-04-22T14:36:03.222306024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:57692"
level=debug ts=2026-04-22T14:36:05.558451819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:36:25.685515109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:36:27.770907816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:33754"
ts=2026-04-22T14:36:55.687121035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f92f37df 10.129.0.72:7946"
level=debug ts=2026-04-22T14:37:05.815936889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:37:25.688580839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:37:27.774369256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41748"
ts=2026-04-22T14:37:43.63812932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:53740"
ts=2026-04-22T14:37:55.690726218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f92f37df 10.129.0.72:7946"
level=debug ts=2026-04-22T14:38:07.171124508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:38:25.692454916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:38:27.77732967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45850"
ts=2026-04-22T14:38:55.694353761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f92f37df 10.129.0.72:7946"
ts=2026-04-22T14:38:57.779251508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36608"
level=debug ts=2026-04-22T14:39:07.525265853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:39:13.642858491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:58014"
ts=2026-04-22T14:39:25.696272404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f92f37df 10.129.0.72:7946"
ts=2026-04-22T14:39:27.780908853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51784"
ts=2026-04-22T14:39:33.231721506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45000"
ts=2026-04-22T14:39:55.698647759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:39:57.782396937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53270"
level=debug ts=2026-04-22T14:40:05.666432331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:40:05.668346475Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:40:13.646665819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:59444"
ts=2026-04-22T14:40:25.7009477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:40:55.702823975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:41:03.235690304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55770"
level=debug ts=2026-04-22T14:41:07.276895901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:41:13.650579656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:51216"
ts=2026-04-22T14:41:25.704915766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:41:27.787314999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51014"
level=debug ts=2026-04-22T14:41:49.574217014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:41:55.706490523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f92f37df 10.129.0.72:7946"
ts=2026-04-22T14:41:57.788944722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41132"
ts=2026-04-22T14:42:03.23864639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49720"
ts=2026-04-22T14:42:25.708283799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:42:33.240982862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41206"
level=debug ts=2026-04-22T14:42:41.798746399Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:42:43.654756956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48468"
ts=2026-04-22T14:42:55.70961214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:43:03.243178788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45148"
level=error ts=2026-04-22T14:43:25.015334949Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:43:25.017339561Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:43:25.019248945Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:43:25.019272058Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:43:25.711504107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f92f37df 10.129.0.72:7946"
ts=2026-04-22T14:43:27.794528277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37280"
ts=2026-04-22T14:43:33.245200183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:56582"
level=debug ts=2026-04-22T14:43:38.932069038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:43:55.713600224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f92f37df 10.129.0.72:7946"
ts=2026-04-22T14:44:25.71587967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T14:44:35.764626368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:44:35.766435704Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:44:40.713940086Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f92f37df (timeout reached)"
ts=2026-04-22T14:44:43.659960635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:58988"
ts=2026-04-22T14:44:43.713977709Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f92f37df has failed, no acks received"
ts=2026-04-22T14:44:46.721019135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34754"
ts=2026-04-22T14:44:46.726329745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34760"
ts=2026-04-22T14:44:53.674548727Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-f92f37df as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T14:44:55.717464089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:45:11.191062366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:45:25.718850761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:45:27.799042396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:38350"
ts=2026-04-22T14:45:55.72062882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:46:02.179638637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:46:25.722982119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:46:55.725453564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:47:01.864582231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:47:25.727284274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:47:55.729084633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:48:03.132834362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:48:12.206179936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40764"
ts=2026-04-22T14:48:25.731213901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c5cbe83d 10.129.0.72:7946"
ts=2026-04-22T14:48:43.671562997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:52292"
ts=2026-04-22T14:48:55.733208415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T14:49:02.923048974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:49:02.924752729Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:49:12.210165134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48220"
ts=2026-04-22T14:49:25.734930112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c5cbe83d 10.129.0.72:7946"
ts=2026-04-22T14:49:27.810460514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41820"
ts=2026-04-22T14:49:55.73683855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:49:57.812530031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35812"
level=debug ts=2026-04-22T14:49:58.494810949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:50:13.675293746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:41868"
ts=2026-04-22T14:50:25.738756709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c5cbe83d 10.129.0.72:7946"
ts=2026-04-22T14:50:43.676864787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:32902"
level=debug ts=2026-04-22T14:50:54.362567872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:50:55.740996748Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-c5cbe83d failed: dial tcp 10.129.0.72:7946: connect: connection refused"
ts=2026-04-22T14:50:57.815212274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49306"
ts=2026-04-22T14:51:00.714261944Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c5cbe83d (timeout reached)"
ts=2026-04-22T14:51:03.713921626Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c5cbe83d has failed, no acks received"
ts=2026-04-22T14:51:13.678330279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:42398"
ts=2026-04-22T14:51:15.714070801Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c5cbe83d (timeout reached)"
ts=2026-04-22T14:51:18.475296123Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-c5cbe83d as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T14:51:18.714016623Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c5cbe83d has failed, no acks received"
ts=2026-04-22T14:51:20.727814759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43402"
ts=2026-04-22T14:51:20.733848241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43418"
ts=2026-04-22T14:51:25.742403938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:51:27.817114988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49530"
ts=2026-04-22T14:51:43.679880258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:51292"
level=debug ts=2026-04-22T14:51:51.993597952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:51:55.744171517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:51:57.819552055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45936"
ts=2026-04-22T14:52:20.450544246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38764"
ts=2026-04-22T14:52:25.746161605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:52:27.820968357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45114"
level=debug ts=2026-04-22T14:52:44.448087974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:52:45.713253921Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-2b0efb7d (timeout reached)"
ts=2026-04-22T14:52:48.71322093Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-2b0efb7d has failed, no acks received"
ts=2026-04-22T14:52:55.747698427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:52:57.823267477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45486"
ts=2026-04-22T14:53:08.476326011Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-2b0efb7d as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-22T14:53:23.471781406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:53:23.473388764Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:53:23.7269084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:47398"
ts=2026-04-22T14:53:23.733412536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:47400"
level=error ts=2026-04-22T14:53:25.012708538Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:53:25.014460254Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:53:25.016116247Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T14:53:25.016160463Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:53:25.749652537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:53:55.751236696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab22c503 10.129.0.72:7946"
ts=2026-04-22T14:53:58.664441203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51910"
level=debug ts=2026-04-22T14:54:00.934249965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:54:25.752843168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab22c503 10.129.0.72:7946"
ts=2026-04-22T14:54:55.754666678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T14:55:01.103506246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:55:25.756895736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab22c503 10.129.0.72:7946"
ts=2026-04-22T14:55:27.830167075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:38926"
ts=2026-04-22T14:55:43.68920166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:36152"
ts=2026-04-22T14:55:55.758859112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T14:55:59.599941466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:56:25.76087546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab22c503 10.129.0.72:7946"
level=debug ts=2026-04-22T14:56:40.596498766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:56:55.762696815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:56:57.834598219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:45428"
ts=2026-04-22T14:56:58.672873428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35422"
level=debug ts=2026-04-22T14:57:23.181922043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T14:57:23.18361178Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T14:57:25.764518822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab22c503 10.129.0.72:7946"
ts=2026-04-22T14:57:55.76622492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:57:58.67601387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42938"
level=debug ts=2026-04-22T14:58:20.057536885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:58:25.76797208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab22c503 10.129.0.72:7946"
ts=2026-04-22T14:58:27.838360227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:55056"
ts=2026-04-22T14:58:28.677976521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54944"
ts=2026-04-22T14:58:43.698245561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:37162"
ts=2026-04-22T14:58:55.770202104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:58:57.840647319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:32976"
ts=2026-04-22T14:58:58.6798764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33262"
level=debug ts=2026-04-22T14:59:06.733415546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:59:13.699879032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:53206"
ts=2026-04-22T14:59:25.714260006Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ab22c503 (timeout reached)"
ts=2026-04-22T14:59:25.771560786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T14:59:28.713937347Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ab22c503 has failed, no acks received"
ts=2026-04-22T14:59:43.701916994Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:37996"
ts=2026-04-22T14:59:48.714056459Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-ab22c503 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-22T14:59:49.442189596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T14:59:50.714712247Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ab22c503 (timeout reached)"
ts=2026-04-22T14:59:53.714003123Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ab22c503 has failed, no acks received"
ts=2026-04-22T14:59:55.773128526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T14:59:57.843186513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41966"
ts=2026-04-22T15:00:25.775028636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T15:00:43.705478327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:43018"
level=debug ts=2026-04-22T15:00:49.094546374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T15:00:49.729600954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53752"
ts=2026-04-22T15:00:49.735976236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53768"
ts=2026-04-22T15:00:55.777452671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T15:00:57.8462548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50468"
ts=2026-04-22T15:01:25.779627038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T15:01:40.705823051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T15:01:40.707620301Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T15:01:43.707932394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:60794"
ts=2026-04-22T15:01:55.781760001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
ts=2026-04-22T15:02:13.710288803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:48726"
level=debug ts=2026-04-22T15:02:20.620250697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T15:02:25.783729286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T15:02:43.712279338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:38268"
ts=2026-04-22T15:02:48.526746858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51982"
ts=2026-04-22T15:02:55.785474804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T15:02:57.400804644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T15:03:18.528511884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36096"
level=error ts=2026-04-22T15:03:25.01370841Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T15:03:25.015474806Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T15:03:25.017292425Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T15:03:25.017327171Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T15:03:25.787936596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5b8e11b4 10.129.0.72:7946"
ts=2026-04-22T15:03:27.852584413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37422"
level=debug ts=2026-04-22T15:03:41.331445279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T15:03:43.71457701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:51232"
ts=2026-04-22T15:03:48.530035268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33818"
ts=2026-04-22T15:03:55.789421724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5b8e11b4 10.129.0.72:7946"
ts=2026-04-22T15:03:57.85448169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:48154"
ts=2026-04-22T15:04:13.716291897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:39618"
ts=2026-04-22T15:04:25.791464663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-fzxtt-9d021435 10.129.0.70:7946"
level=debug ts=2026-04-22T15:04:33.678928267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T15:04:55.79294172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T15:05:13.719519564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:50450"
ts=2026-04-22T15:05:25.795222977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5b8e11b4 10.129.0.72:7946"
level=debug ts=2026-04-22T15:05:32.114276792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T15:05:32.115800548Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T15:05:55.797271969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
ts=2026-04-22T15:05:57.860041978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50452"
level=debug ts=2026-04-22T15:06:20.585801238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T15:06:25.798996866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-5b8e11b4 10.129.0.72:7946"
ts=2026-04-22T15:06:55.801502777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-tsgps-f94462f7 10.129.0.74:7946"
level=debug ts=2026-04-22T15:06:59.942542856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T15:07:13.725807501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:37260"
ts=2026-04-22T15:07:18.540913293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38342"