level=info ts=2026-05-12T02:36:48.621877475Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-12T02:36:48.621909546Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-12T02:36:48.624512646Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-12T02:36:48.624926275Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T02:36:48.62504213Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.93 inf=eth0
level=debug ts=2026-05-12T02:36:48.625587061Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T02:36:48.625664103Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.93 inf=eth0
level=info ts=2026-05-12T02:36:48.625801012Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-73e01042
ts=2026-05-12T02:36:48.625874925Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-12T02:36:48.625888272Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.93 advertisePort=7946
level=debug ts=2026-05-12T02:36:48.625903734Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.93 advertisePort=7946
level=info ts=2026-05-12T02:36:48.626041533Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-12T02:36:48.633852077Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T02:36:48.633870744Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T02:36:48.633882631Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.842545ms
level=error ts=2026-05-12T02:36:48.6338952Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-12T02:36:48.63391153Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-12T02:36:48.639943236Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T02:36:48.639995565Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-12T02:36:48.648178452Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-12T02:36:48.648214128Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-12T02:36:48.648214991Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-12T02:36:48.648239182Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-12T02:36:48.648248717Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-12T02:36:48.648258611Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-12T02:36:48.648275148Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-12T02:36:48.648537124Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-12T02:36:48.648549315Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-12T02:36:48.648553085Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-12T02:36:48.64861409Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-12T02:36:48.648650236Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-12T02:36:48.648683086Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-12T02:36:48.648840345Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-12T02:36:48.648867132Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-12T02:36:48.648892063Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-12T02:36:49.65002219Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-05-12T02:36:49.655788521Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T02:36:49.655807158Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-12T02:36:49.757044308Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-12T02:36:49.757094207Z caller=loki.go:615 msg="Loki started" startup_time=1.210214186s
ts=2026-05-12T02:36:51.928846987Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T02:36:51.928869511Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-12T02:36:54.757824833Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-12T02:36:54.75786278Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-12T02:36:59.676514846Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.61:7946: dial tcp 10.129.0.61:7946: connect: connection refused"
level=warn ts=2026-05-12T02:36:59.676537125Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to join 10.129.0.61:7946: dial tcp 10.129.0.61:7946: connect: connection refused\n\n"
level=debug ts=2026-05-12T02:37:09.542811653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T02:37:10.199042495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34200"
ts=2026-05-12T02:37:11.004027327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48196"
ts=2026-05-12T02:37:11.722702297Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.61:7946: dial tcp 10.129.0.61:7946: connect: connection refused"
ts=2026-05-12T02:37:11.723425602Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.85:7946: dial tcp 10.128.0.85:7946: connect: connection refused"
ts=2026-05-12T02:37:11.723608342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.86:7946"
ts=2026-05-12T02:37:11.725175211Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.87:7946: dial tcp 10.128.0.87:7946: connect: connection refused"
ts=2026-05-12T02:37:11.725317786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.93:7946"
ts=2026-05-12T02:37:11.725415224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:47422"
ts=2026-05-12T02:37:11.726074619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.90:7946"
level=info ts=2026-05-12T02:37:11.726542139Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=23.092624588s
ts=2026-05-12T02:37:14.206439509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46468"
level=debug ts=2026-05-12T02:37:21.812465454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T02:37:40.2631455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38588"
level=debug ts=2026-05-12T02:37:44.877980626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T02:37:45.032663176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:37:59.058697104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52572"
level=debug ts=2026-05-12T02:38:03.99983469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T02:38:10.264905811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56966"
ts=2026-05-12T02:38:15.03391466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:38:33.976114057Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T02:38:33.977747203Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T02:38:45.035590806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T02:38:59.061626895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:40648"
ts=2026-05-12T02:39:15.037097376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:39:19.877126101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T02:39:27.121627218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46432"
ts=2026-05-12T02:39:45.038765257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:39:57.123288351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42472"
ts=2026-05-12T02:40:15.039891433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:40:21.590985557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T02:40:40.272467426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59072"
ts=2026-05-12T02:40:45.041692049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:40:57.12578599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:36290"
ts=2026-05-12T02:40:59.06637001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:34544"
ts=2026-05-12T02:41:10.273782355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60122"
ts=2026-05-12T02:41:15.042856608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:41:30.805678874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T02:41:40.275411926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56806"
ts=2026-05-12T02:41:45.044791751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T02:41:57.128093712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34090"
ts=2026-05-12T02:42:10.277250763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51354"
ts=2026-05-12T02:42:15.046551882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T02:42:27.129730983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58190"
ts=2026-05-12T02:42:29.070699644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:54936"
level=debug ts=2026-05-12T02:42:34.783027617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T02:42:45.048104657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:42:59.071557414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43234"
ts=2026-05-12T02:43:15.049328128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T02:43:26.294045039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T02:43:26.295461986Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T02:43:27.131843612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:37978"
ts=2026-05-12T02:43:29.072716623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:40090"
ts=2026-05-12T02:43:45.050977967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T02:43:59.073993024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:37052"
level=debug ts=2026-05-12T02:44:06.482181141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:44:15.052654662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:44:29.074994961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:38840"
ts=2026-05-12T02:44:40.283356389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42802"
ts=2026-05-12T02:44:45.054183653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T02:44:48.921503331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:45:15.055594417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:45:35.577431316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:45:45.057309088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T02:45:57.138792721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42720"
ts=2026-05-12T02:46:15.058598636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T02:46:20.870940631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:46:27.140007718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42168"
ts=2026-05-12T02:46:45.059466483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=info ts=2026-05-12T02:46:54.758155693Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-12T02:46:54.761629551Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T02:46:54.763338934Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T02:46:54.765020678Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T02:46:54.765046479Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T02:46:54.765058744Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-12T02:47:13.289171013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T02:47:13.290897357Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T02:47:15.060269155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T02:47:40.290611565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49684"
ts=2026-05-12T02:47:45.061506818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:47:48.658568814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:47:57.143567145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46392"
ts=2026-05-12T02:47:59.08442641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:60270"
ts=2026-05-12T02:48:15.062613364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T02:48:29.086025493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:49082"
level=debug ts=2026-05-12T02:48:34.885428577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:48:40.292883519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39254"
ts=2026-05-12T02:48:45.063793817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:48:59.086939038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55058"
ts=2026-05-12T02:49:10.294428695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58702"
ts=2026-05-12T02:49:15.064676318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:49:19.8577812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:49:45.066528845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:50:08.443242342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:50:15.067919127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:50:45.069287796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:50:57.855156168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T02:50:57.856986029Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T02:51:15.07035077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:51:38.005742769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:51:40.301886317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52898"
ts=2026-05-12T02:51:45.071760173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T02:52:15.073435206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:52:25.388589549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:52:29.096433422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:45822"
ts=2026-05-12T02:52:45.075183577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:52:59.098304635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43172"
ts=2026-05-12T02:53:15.076435688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:53:23.175208955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:53:27.157829336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46740"
ts=2026-05-12T02:53:40.308024995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59636"
ts=2026-05-12T02:53:45.077668053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T02:53:57.159111983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35212"
level=debug ts=2026-05-12T02:54:08.878406713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:54:15.078943667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T02:54:29.101993433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47214"
ts=2026-05-12T02:54:45.080449979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T02:54:56.935374592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T02:54:56.937320951Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T02:54:57.1619431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:54414"
ts=2026-05-12T02:55:10.312719521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52528"
ts=2026-05-12T02:55:15.081828056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T02:55:42.41941289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:55:45.083667668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T02:55:59.105861525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58086"
ts=2026-05-12T02:56:15.085077413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:56:24.925414344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:56:40.318161468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45370"
ts=2026-05-12T02:56:45.086506857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=error ts=2026-05-12T02:56:54.770497157Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T02:56:54.773417349Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T02:56:54.775852485Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T02:56:54.775884432Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T02:57:10.319873525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52980"
ts=2026-05-12T02:57:15.087330542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:57:18.260154357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:57:27.169392323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56456"
ts=2026-05-12T02:57:29.110735687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58178"
ts=2026-05-12T02:57:45.089226065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T02:58:12.248619208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T02:58:15.090319477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T02:58:27.172777627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:57004"
ts=2026-05-12T02:58:29.113259353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:54024"
ts=2026-05-12T02:58:40.323833551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54810"
ts=2026-05-12T02:58:45.092124949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T02:58:59.114918893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41936"
level=debug ts=2026-05-12T02:59:12.962218654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T02:59:12.96407834Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T02:59:15.09326738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T02:59:27.175251551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:56812"
ts=2026-05-12T02:59:45.09502687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T02:59:59.118007392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58064"
level=debug ts=2026-05-12T03:00:06.516170415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:00:15.096474538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:00:29.118989072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43052"
ts=2026-05-12T03:00:40.329971525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59100"
ts=2026-05-12T03:00:45.097788457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:00:57.179935337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35498"
ts=2026-05-12T03:00:59.120213342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48040"
level=debug ts=2026-05-12T03:01:07.985232899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:01:10.331256549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56864"
ts=2026-05-12T03:01:15.098899163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:01:27.181772142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59188"
ts=2026-05-12T03:01:29.121237533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:42022"
ts=2026-05-12T03:01:40.332454374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39170"
ts=2026-05-12T03:01:45.101008692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:01:50.945346416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:01:57.183725991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44688"
ts=2026-05-12T03:02:15.102846724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:02:27.185303151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33212"
ts=2026-05-12T03:02:29.124326721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:51558"
ts=2026-05-12T03:02:45.104165808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:02:45.138947607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:02:59.125367671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39840"
ts=2026-05-12T03:03:15.105185493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:03:25.811191928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:03:25.812813533Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:03:40.338496175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51958"
ts=2026-05-12T03:03:45.10706044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:03:57.189060784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38470"
ts=2026-05-12T03:04:15.107982114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:04:20.637442324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:04:45.108939062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:04:57.192534596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59222"
ts=2026-05-12T03:05:15.110544562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:05:16.186491273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:05:29.132975601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:38650"
ts=2026-05-12T03:05:45.111839095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:05:57.47107017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:05:59.134898708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:32910"
ts=2026-05-12T03:06:15.113195315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:06:29.136508455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55842"
level=debug ts=2026-05-12T03:06:41.049401863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:06:45.114422528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=error ts=2026-05-12T03:06:54.770032416Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:06:54.772213893Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:06:54.774347098Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:06:54.774375014Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:06:57.197789029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46542"
ts=2026-05-12T03:07:15.115699592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:07:23.380252773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:07:23.381750083Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:07:29.140653031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:51430"
ts=2026-05-12T03:07:40.352042298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41502"
ts=2026-05-12T03:07:45.117431399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:08:10.353074149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49550"
level=debug ts=2026-05-12T03:08:13.056221754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:08:15.118419516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:08:27.202555593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44734"
ts=2026-05-12T03:08:29.147795679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:38964"
ts=2026-05-12T03:08:45.119701214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:09:11.852524269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:09:15.121536513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:09:45.12307492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:10:09.814222639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:10:15.124290719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:10:27.207717726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42800"
ts=2026-05-12T03:10:40.361835189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53880"
ts=2026-05-12T03:10:45.125341508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:10:55.052774101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:11:10.363219825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34988"
ts=2026-05-12T03:11:15.126709736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:11:27.210980174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:55840"
ts=2026-05-12T03:11:29.157267499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41534"
ts=2026-05-12T03:11:45.128574005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:11:52.025044072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:11:52.02680335Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:11:59.159039898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46062"
ts=2026-05-12T03:12:10.366381727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40228"
ts=2026-05-12T03:12:15.129845506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:12:29.160621881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:40244"
level=debug ts=2026-05-12T03:12:42.447711098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:12:45.131617884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:13:10.369757734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42000"
ts=2026-05-12T03:13:15.133042224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:13:36.123655882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:13:40.371660204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57324"
ts=2026-05-12T03:13:45.13408227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:13:57.21872595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52554"
ts=2026-05-12T03:14:15.135088646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:14:24.478974871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:14:45.137011758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:14:57.222208814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41826"
ts=2026-05-12T03:15:10.37533355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43144"
level=debug ts=2026-05-12T03:15:11.371119445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:15:15.138569868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:15:45.139566574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:15:54.462542125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:15:54.46436368Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:15:59.170428177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53516"
ts=2026-05-12T03:16:15.140455945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:16:39.441512671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:16:45.142088385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=error ts=2026-05-12T03:16:54.770998635Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:16:54.772846714Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:16:54.775078488Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:16:54.775110956Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:17:15.144000909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:17:30.659592665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:17:45.145986075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:17:59.175998283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:50700"
ts=2026-05-12T03:18:15.148472721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:18:23.27712882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:18:27.23422128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:57784"
ts=2026-05-12T03:18:29.177793701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:36696"
ts=2026-05-12T03:18:40.383380268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42512"
ts=2026-05-12T03:18:45.150298459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:19:10.385180815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59580"
ts=2026-05-12T03:19:15.151385034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:19:21.885808538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:19:27.238376359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:41894"
ts=2026-05-12T03:19:45.15259714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:19:57.239892477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40854"
level=debug ts=2026-05-12T03:20:02.932728466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:20:02.934515543Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:20:15.154364154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:20:37.504359564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:20:45.155763901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:21:15.157557805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:21:34.166321803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:21:45.159657024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:21:57.244762179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:50340"
ts=2026-05-12T03:22:15.161156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:22:27.246214246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:48632"
level=debug ts=2026-05-12T03:22:31.996736762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:22:45.162480256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:23:10.396227282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55752"
ts=2026-05-12T03:23:15.16355011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:23:27.249361565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42398"
level=debug ts=2026-05-12T03:23:33.183147617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:23:45.165417205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:23:57.251220787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39916"
ts=2026-05-12T03:24:10.399045344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38488"
ts=2026-05-12T03:24:15.167226142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:24:27.25254283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:47176"
level=debug ts=2026-05-12T03:24:27.613548628Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:24:27.615105796Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:24:40.400471003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59176"
ts=2026-05-12T03:24:45.169235626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:24:57.254294105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:50416"
ts=2026-05-12T03:25:15.170386254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:25:24.249234607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:25:27.255737121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33654"
ts=2026-05-12T03:25:45.171205457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:25:57.257122024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59984"
level=debug ts=2026-05-12T03:26:02.53022025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:26:15.172493779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:26:29.198902253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:59596"
level=debug ts=2026-05-12T03:26:39.27664929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:26:45.173552922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=error ts=2026-05-12T03:26:54.769128281Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:26:54.770844308Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:26:54.772432208Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:26:54.772453044Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:27:15.17476987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:27:15.739400685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:27:45.176158341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:27:57.262627808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39230"
ts=2026-05-12T03:27:59.202168555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46068"
ts=2026-05-12T03:28:10.409180477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50016"
level=debug ts=2026-05-12T03:28:13.619288646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:28:13.62134255Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:28:15.177477443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:28:45.179110783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:28:59.204622981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:57688"
level=debug ts=2026-05-12T03:28:59.443526687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:29:15.179915653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:29:27.266599701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59178"
ts=2026-05-12T03:29:29.205902033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41622"
ts=2026-05-12T03:29:45.181646451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:29:51.713539578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:29:57.268273984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:34854"
ts=2026-05-12T03:29:59.206993036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46796"
ts=2026-05-12T03:30:10.41484512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55868"
ts=2026-05-12T03:30:15.183388634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:30:29.20790003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48632"
ts=2026-05-12T03:30:40.415884866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38298"
ts=2026-05-12T03:30:45.184462955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:30:50.615985978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:31:15.185990206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:31:31.216444573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:31:40.419114236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54274"
ts=2026-05-12T03:31:45.186886088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:31:59.212246633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:34662"
ts=2026-05-12T03:32:10.420308316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56084"
ts=2026-05-12T03:32:15.188623668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:32:25.993645083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:32:25.99548339Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:32:29.214004302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:38688"
ts=2026-05-12T03:32:40.421922437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51658"
ts=2026-05-12T03:32:45.190148477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:33:07.291387884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:33:15.191708644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:33:29.216611429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:50764"
ts=2026-05-12T03:33:45.193227716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:33:59.217564116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47472"
level=debug ts=2026-05-12T03:34:03.376331386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:34:10.425729742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53664"
ts=2026-05-12T03:34:15.194927263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:34:27.281729259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59578"
ts=2026-05-12T03:34:40.42700169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42240"
ts=2026-05-12T03:34:45.196772711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:34:56.710683324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:35:15.19826606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:35:27.28458667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33386"
ts=2026-05-12T03:35:40.429480399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47632"
level=debug ts=2026-05-12T03:35:43.50386377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:35:45.199100356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:35:59.222929142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55554"
ts=2026-05-12T03:36:15.200909248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:36:29.224118891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47120"
ts=2026-05-12T03:36:40.432033812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34672"
level=debug ts=2026-05-12T03:36:41.297785678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:36:41.299977152Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:36:45.202223674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=error ts=2026-05-12T03:36:54.768655528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:36:54.770490118Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:36:54.772263389Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:36:54.772287751Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:37:15.20373796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:37:17.862654815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:37:27.288987892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:35618"
ts=2026-05-12T03:37:40.434555615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52762"
ts=2026-05-12T03:37:45.20535613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:37:59.228535972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55694"
level=debug ts=2026-05-12T03:38:00.279048506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:38:10.435823215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56592"
ts=2026-05-12T03:38:15.206592853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:38:29.230353672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39948"
ts=2026-05-12T03:38:40.438000813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52710"
ts=2026-05-12T03:38:45.208162713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:38:51.904122012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:39:15.209856516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:39:29.931087455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:39:45.211764777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:40:08.18387357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:40:08.186002216Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:40:15.212639823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:40:27.296221249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58590"
ts=2026-05-12T03:40:45.21463219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:40:48.438256951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:41:15.215688149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:41:23.098032208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:41:45.217444088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:41:57.29962976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:42424"
ts=2026-05-12T03:42:15.219184912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:42:20.336970237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:42:27.301239807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38780"
ts=2026-05-12T03:42:45.220785571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:42:57.303053963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:33214"
ts=2026-05-12T03:42:59.242527132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:60266"
level=debug ts=2026-05-12T03:43:12.479574153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:43:15.222339006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:43:29.243428839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43782"
ts=2026-05-12T03:43:40.452360359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55254"
ts=2026-05-12T03:43:45.223333018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:43:59.245279792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39290"
level=debug ts=2026-05-12T03:44:13.319271699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:44:13.320806098Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:44:15.224753847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:44:45.225916708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:44:59.129362802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:45:15.22681204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:45:27.309509107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59548"
ts=2026-05-12T03:45:40.457456399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37448"
ts=2026-05-12T03:45:45.227679246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:45:53.03084065Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:45:59.251031523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52764"
ts=2026-05-12T03:46:15.228731894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:46:29.251910634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:36326"
ts=2026-05-12T03:46:40.460409127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35974"
ts=2026-05-12T03:46:45.230402302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:46:50.94093994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:46:54.767617996Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:46:54.769204012Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:46:54.772442475Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:46:54.772466169Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:46:57.313807274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:60384"
ts=2026-05-12T03:47:15.231842827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:47:27.315625806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46120"
ts=2026-05-12T03:47:29.255535617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:54900"
ts=2026-05-12T03:47:45.233632296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:47:51.27000524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:47:57.317261411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:38130"
ts=2026-05-12T03:47:59.257308469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58824"
ts=2026-05-12T03:48:15.235128811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:48:45.23618448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:48:52.509048123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:48:52.510763757Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:49:10.468037861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33472"
ts=2026-05-12T03:49:15.237437627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:49:27.321742354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:59840"
ts=2026-05-12T03:49:29.261834476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47086"
level=debug ts=2026-05-12T03:49:36.611910418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:49:45.23899079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:49:57.322819824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40616"
ts=2026-05-12T03:50:15.24017067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T03:50:24.72557518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:50:45.241691125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:50:57.325046561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:60010"
ts=2026-05-12T03:50:59.265523175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:49276"
ts=2026-05-12T03:51:15.24330567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:51:18.364734142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:51:29.266705646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43258"
ts=2026-05-12T03:51:40.476116144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43462"
ts=2026-05-12T03:51:45.244701056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:51:57.326638264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:53730"
level=debug ts=2026-05-12T03:52:02.093558588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:52:15.24569195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:52:40.265922686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:52:40.267446856Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:52:45.246880281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:53:15.247934189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T03:53:27.330195699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45130"
level=debug ts=2026-05-12T03:53:41.09148608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:53:45.248803628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:53:57.331651621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:43266"
ts=2026-05-12T03:54:10.483124318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44286"
ts=2026-05-12T03:54:15.250441308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:54:29.275144953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46496"
level=debug ts=2026-05-12T03:54:34.439996904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:54:45.251312617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:54:59.276213172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:40022"
ts=2026-05-12T03:55:15.252681922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:55:31.973359527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:55:45.253789557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T03:56:15.2552395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:56:23.141702967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:56:45.256453671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=error ts=2026-05-12T03:56:54.768895947Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:56:54.77121789Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:56:54.773327867Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T03:56:54.773354077Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:56:59.281702128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52974"
ts=2026-05-12T03:57:15.257596738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T03:57:22.57242739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T03:57:22.574025412Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T03:57:45.258473169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:58:15.260138545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:58:22.137768627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:58:45.261502487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T03:58:56.978747054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:58:59.288578465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52732"
ts=2026-05-12T03:59:10.49935743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58340"
ts=2026-05-12T03:59:15.262936088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T03:59:27.34732767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:32778"
ts=2026-05-12T03:59:29.289879371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58342"
ts=2026-05-12T03:59:40.501351248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52098"
level=debug ts=2026-05-12T03:59:44.873869276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T03:59:45.264289766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T04:00:15.26599462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:00:20.47377645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:00:27.350627805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:52308"
ts=2026-05-12T04:00:29.292269169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:56534"
ts=2026-05-12T04:00:45.267458425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:00:56.015816026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T04:00:56.017751923Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T04:01:15.268457305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T04:01:29.295010933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41822"
ts=2026-05-12T04:01:45.269863769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:01:48.410897231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:01:59.296683055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53552"
ts=2026-05-12T04:02:15.270877784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:02:36.747247202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:02:45.272395074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T04:03:15.273570631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T04:03:19.87576907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:03:45.274582978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:03:54.41790613Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:04:15.275900738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T04:04:29.30388927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48740"
ts=2026-05-12T04:04:45.277629172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:04:50.356740073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T04:04:50.358257656Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T04:05:10.51733333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52446"
ts=2026-05-12T04:05:15.279010454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T04:05:45.279773375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T04:05:46.579634038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:06:15.280934715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T04:06:45.282549107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:06:46.662818469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T04:06:54.769728866Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T04:06:54.771543178Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T04:06:54.773190114Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T04:06:54.77321445Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T04:06:59.31147217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39700"
ts=2026-05-12T04:07:10.523077866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57928"
ts=2026-05-12T04:07:15.284356574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:07:28.303767475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:07:45.285977615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T04:08:10.525829812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49656"
ts=2026-05-12T04:08:15.288023078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:08:21.293713519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:08:45.289467488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T04:09:15.290831071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:09:22.369498508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T04:09:22.371087201Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T04:09:27.377397078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:58384"
ts=2026-05-12T04:09:45.292488453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T04:09:59.321528422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:42392"
ts=2026-05-12T04:10:15.293514452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:10:16.844825129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:10:27.379986881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46230"
ts=2026-05-12T04:10:45.294911932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:10:57.305555261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:10:57.381358859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:44034"
ts=2026-05-12T04:10:59.323514996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:49246"
ts=2026-05-12T04:11:10.53315809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42052"
ts=2026-05-12T04:11:15.295816334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T04:11:27.382804429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:39624"
ts=2026-05-12T04:11:29.325255777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48438"
level=debug ts=2026-05-12T04:11:42.121195639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:11:45.296819366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T04:12:15.297735075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T04:12:18.552939054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:12:27.385585773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46002"
ts=2026-05-12T04:12:40.537419594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41070"
ts=2026-05-12T04:12:45.299528871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T04:13:15.30068551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:13:19.265124898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T04:13:19.26757214Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T04:13:45.302499976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
level=debug ts=2026-05-12T04:14:05.295398748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:14:15.303602609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T04:14:29.333434269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55978"
ts=2026-05-12T04:14:45.305157209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:14:48.064172126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:14:57.391909037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40918"
ts=2026-05-12T04:14:59.334763616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:56700"
ts=2026-05-12T04:15:15.306568236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T04:15:27.393519956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:57144"
ts=2026-05-12T04:15:29.335754208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:49296"
level=debug ts=2026-05-12T04:15:32.761697843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:15:40.545268526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45448"
ts=2026-05-12T04:15:45.308398691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T04:15:57.39487966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:40786"
ts=2026-05-12T04:16:15.309648812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:16:27.090356578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:16:45.311444655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=error ts=2026-05-12T04:16:54.769555792Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T04:16:54.771512731Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T04:16:54.773576166Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T04:16:54.773601549Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T04:17:10.550099577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41656"
ts=2026-05-12T04:17:15.312589454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:17:27.523926225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T04:17:27.525644349Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T04:17:29.342411106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52596"
ts=2026-05-12T04:17:45.31435729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:18:08.656825073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:18:15.316200988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
ts=2026-05-12T04:18:29.347469842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53944"
ts=2026-05-12T04:18:45.31772228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:18:46.235537531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:19:15.319476039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:19:34.120159036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:19:40.562594377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45972"
ts=2026-05-12T04:19:45.320680656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-b9cwq-967753fe 10.128.0.90:7946"
ts=2026-05-12T04:19:57.405411934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:51030"
ts=2026-05-12T04:20:15.321493835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
level=debug ts=2026-05-12T04:20:22.351403014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:20:45.322449097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:21:10.586768763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T04:21:10.588356525Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T04:21:15.323834408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-90719b26 10.128.0.92:7946"
ts=2026-05-12T04:21:27.409484323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:45086"
ts=2026-05-12T04:21:45.325024304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4cjv6-6d49ff24 10.128.0.86:7946"
level=debug ts=2026-05-12T04:21:53.344512111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T04:21:57.410918942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.90:46780"