level=info ts=2026-05-18T11:25:16.328136459Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-18T11:25:16.32818265Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-18T11:25:16.329505665Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-18T11:25:16.329850426Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-18T11:25:16.329921353Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.76 inf=eth0
level=debug ts=2026-05-18T11:25:16.330859463Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-18T11:25:16.330918581Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.76 inf=eth0
level=info ts=2026-05-18T11:25:16.330988449Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-15650c0d
ts=2026-05-18T11:25:16.331078618Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-18T11:25:16.331091322Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.76 advertisePort=7946
level=debug ts=2026-05-18T11:25:16.331107075Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.76 advertisePort=7946
level=info ts=2026-05-18T11:25:16.331234861Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-18T11:25:16.338796468Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-18T11:25:16.338812269Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-18T11:25:16.338822852Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.590509ms
level=error ts=2026-05-18T11:25:16.338831479Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-18T11:25:16.338842518Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-18T11:25:16.34126322Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T11:25:16.341275868Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-18T11:25:16.346112801Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-18T11:25:16.346281876Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-18T11:25:16.346294987Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-18T11:25:16.346321742Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-18T11:25:16.346339473Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-18T11:25:16.346416397Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-18T11:25:16.346640957Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-18T11:25:16.346677527Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-18T11:25:16.346683107Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-18T11:25:16.346688216Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-18T11:25:16.346748595Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-18T11:25:16.346790065Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-18T11:25:16.346812602Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-18T11:25:16.346975368Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-18T11:25:16.346988523Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-18T11:25:16.34700882Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-18T11:25:17.347813118Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-05-18T11:25:17.490928991Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T11:25:17.490955575Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-18T11:25:17.517048244Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-18T11:25:17.517105294Z caller=loki.go:615 msg="Loki started" startup_time=1.24683279s
ts=2026-05-18T11:25:21.084419604Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T11:25:21.084441606Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-18T11:25:22.517517081Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-18T11:25:22.517568568Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-18T11:25:27.929724163Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-18T11:25:27.929747921Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-18T11:25:32.786406093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T11:25:35.967149326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47348"
ts=2026-05-18T11:25:37.278121712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60406"
ts=2026-05-18T11:25:37.70281427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:37290"
ts=2026-05-18T11:25:41.360065551Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.75:7946: dial tcp 10.129.0.75:7946: connect: connection refused"
ts=2026-05-18T11:25:41.360174634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.76:7946"
ts=2026-05-18T11:25:41.36019401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58092"
ts=2026-05-18T11:25:41.36171346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-05-18T11:25:41.364268498Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.74:7946: dial tcp 10.130.0.74:7946: connect: connection refused"
ts=2026-05-18T11:25:41.365228059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.73:7946"
ts=2026-05-18T11:25:41.367675763Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.71:7946: dial tcp 10.130.0.71:7946: connect: connection refused"
level=info ts=2026-05-18T11:25:41.367685859Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=25.028837652s
level=debug ts=2026-05-18T11:25:49.041064178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T11:25:54.981471923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41594"
ts=2026-05-18T11:25:57.529159534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:26:09.445377282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T11:26:12.443114496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58866"
ts=2026-05-18T11:26:21.988435828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:42738"
level=debug ts=2026-05-18T11:26:27.501129689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T11:26:27.531192307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:26:51.77579403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-18T11:26:51.777283386Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:26:57.534772115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:27:27.536698657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:27:42.449977365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38844"
level=debug ts=2026-05-18T11:27:43.725831402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T11:27:51.996339729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:56682"
ts=2026-05-18T11:27:57.539052475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:28:27.541637247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:28:42.455685444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:44290"
level=debug ts=2026-05-18T11:28:52.572983069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T11:28:57.543427174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:29:24.999085776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:56958"
ts=2026-05-18T11:29:27.546018315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:29:46.671361202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T11:29:52.006567705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:42316"
ts=2026-05-18T11:29:55.002483786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44638"
ts=2026-05-18T11:29:57.549538241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:30:12.464739834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58516"
ts=2026-05-18T11:30:27.551805264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:30:39.108870052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-18T11:30:57.554053564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:31:27.556941759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:31:42.472236205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:35512"
level=debug ts=2026-05-18T11:31:43.225227237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-18T11:31:43.227064422Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:31:57.558105855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:32:12.475759467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34454"
ts=2026-05-18T11:32:27.560711598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:32:39.346277978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:32:55.016057991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41178"
ts=2026-05-18T11:32:57.562455335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:33:22.027581304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58540"
ts=2026-05-18T11:33:27.56489247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:33:39.682397475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:33:42.486126051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:41606"
ts=2026-05-18T11:33:57.567480783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:34:27.569229752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:34:35.306768892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:34:55.026182741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54356"
ts=2026-05-18T11:34:57.57167935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:35:10.601237699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:35:12.491730786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:43004"
level=info ts=2026-05-18T11:35:22.518272112Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-18T11:35:22.522446728Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:35:22.524346341Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:35:22.526196285Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:35:22.526224619Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-18T11:35:22.526237996Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-18T11:35:27.573528904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:35:49.250821548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T11:35:49.252358775Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:35:52.041931366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:37036"
ts=2026-05-18T11:35:55.030115384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:59406"
ts=2026-05-18T11:35:57.576142433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:36:22.043430065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59624"
ts=2026-05-18T11:36:25.033662067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33706"
ts=2026-05-18T11:36:27.579744454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:36:35.706101014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:36:57.582980749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:37:17.171515472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:37:27.586430373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T11:37:52.051547338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:56564"
ts=2026-05-18T11:37:57.589949744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:37:59.14849152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:38:12.50956509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53954"
ts=2026-05-18T11:38:27.59333148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T11:38:55.047908284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53964"
level=debug ts=2026-05-18T11:38:57.580403227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:38:57.595686451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:39:25.051605778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33472"
ts=2026-05-18T11:39:27.597378891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:39:48.929818566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T11:39:48.931410708Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:39:57.59940098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:40:26.915947466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:40:27.600570443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:40:52.067435791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:44072"
ts=2026-05-18T11:40:57.602593054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:41:13.05231206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:41:27.605471204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:41:54.986581422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:41:57.608955199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T11:42:12.529580883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:44588"
ts=2026-05-18T11:42:27.611361267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:42:29.161120404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:42:55.067715587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47210"
ts=2026-05-18T11:42:57.613186852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:43:27.614198676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:43:28.442414049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T11:43:28.443961424Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:43:42.536494827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42748"
ts=2026-05-18T11:43:55.074898122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44016"
ts=2026-05-18T11:43:57.616885571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:44:12.756633042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:44:27.619222005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:44:42.541878702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57542"
ts=2026-05-18T11:44:52.090164565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38934"
ts=2026-05-18T11:44:55.079810878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37988"
ts=2026-05-18T11:44:57.621879491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:45:01.805518963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:45:12.545460486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58360"
level=error ts=2026-05-18T11:45:22.531002331Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:45:22.53291486Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:45:22.534787096Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:45:22.534817199Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:45:25.083264539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54782"
ts=2026-05-18T11:45:27.624585083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:45:57.628195644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:46:01.272892797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:46:22.098357003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:35536"
ts=2026-05-18T11:46:27.630616152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:46:47.13850926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:46:57.632382813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:47:22.102618178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39220"
level=debug ts=2026-05-18T11:47:23.496837909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T11:47:23.498600254Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:47:27.634304504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:47:55.094377405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43990"
ts=2026-05-18T11:47:57.637286213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:48:12.555230695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:35784"
level=debug ts=2026-05-18T11:48:21.945635231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:48:22.106922379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:36090"
ts=2026-05-18T11:48:25.097383409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38624"
ts=2026-05-18T11:48:27.64047631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T11:48:52.107748852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:60714"
ts=2026-05-18T11:48:57.643121468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:49:17.188104384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:49:25.101483493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33040"
ts=2026-05-18T11:49:27.645009569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:49:55.104293476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38042"
ts=2026-05-18T11:49:57.646111058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:50:02.749616235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:50:12.563727361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59886"
ts=2026-05-18T11:50:22.115154112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38956"
ts=2026-05-18T11:50:27.647579073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:50:42.567138501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:46110"
ts=2026-05-18T11:50:52.11658828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57982"
ts=2026-05-18T11:50:55.111052844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37172"
ts=2026-05-18T11:50:57.649556824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:50:58.938919396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:51:12.570572229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36684"
ts=2026-05-18T11:51:22.118463845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:60398"
ts=2026-05-18T11:51:25.113826002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54198"
ts=2026-05-18T11:51:27.653051468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T11:51:57.655074963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:51:59.126209134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T11:51:59.127778663Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:52:12.574912019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56024"
ts=2026-05-18T11:52:27.656794791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:52:52.125638047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:44926"
ts=2026-05-18T11:52:57.659500989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:52:57.667872378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:53:12.5796801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60544"
ts=2026-05-18T11:53:22.12647931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57638"
ts=2026-05-18T11:53:27.662719825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:53:31.780664228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:53:57.665599496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:54:18.673086773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:54:27.669413173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T11:54:42.588303963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:49350"
ts=2026-05-18T11:54:55.132768657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54602"
ts=2026-05-18T11:54:57.672728474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T11:55:08.053187547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:55:22.137964188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52246"
level=error ts=2026-05-18T11:55:22.531061096Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:55:22.532750194Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:55:22.534699636Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T11:55:22.534723989Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:55:25.13522011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54020"
ts=2026-05-18T11:55:27.674606601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:55:52.138885176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46378"
ts=2026-05-18T11:55:55.138099259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36038"
ts=2026-05-18T11:55:57.676502847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:56:07.08783948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T11:56:07.089364113Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T11:56:12.59715919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42128"
ts=2026-05-18T11:56:27.679011915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T11:56:41.685202691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:56:42.600303108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59942"
ts=2026-05-18T11:56:52.142341765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57196"
ts=2026-05-18T11:56:57.679994195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:57:12.603307158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34918"
ts=2026-05-18T11:57:27.682103541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T11:57:42.570033859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:57:52.146747333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34520"
ts=2026-05-18T11:57:55.148949033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52250"
ts=2026-05-18T11:57:57.685014514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:58:22.14830519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59452"
level=debug ts=2026-05-18T11:58:23.873811099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:58:25.151791389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:55082"
ts=2026-05-18T11:58:27.687158166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T11:58:52.149268576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:42782"
ts=2026-05-18T11:58:55.154420326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35692"
ts=2026-05-18T11:58:57.689077533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:59:12.614449943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:54252"
level=debug ts=2026-05-18T11:59:22.22748746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T11:59:27.692486288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T11:59:57.695325393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:00:12.620378593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50018"
level=debug ts=2026-05-18T12:00:12.846626463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:00:12.848736938Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:00:27.698844111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:00:49.789545958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:00:52.15931194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38396"
ts=2026-05-18T12:00:57.70237515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:01:22.160430976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38824"
ts=2026-05-18T12:01:27.705612741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:01:42.630012601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51592"
level=debug ts=2026-05-18T12:01:43.097530988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:01:52.161598979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59652"
ts=2026-05-18T12:01:57.70824217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:02:27.710332097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:02:45.417571477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:02:57.713375018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:03:12.638674409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:54472"
ts=2026-05-18T12:03:22.169376185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55338"
level=debug ts=2026-05-18T12:03:23.655976824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:03:27.716999428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:03:42.64123065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56626"
ts=2026-05-18T12:03:52.17129727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46858"
ts=2026-05-18T12:03:57.720595636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:04:06.753007581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:04:06.754865513Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:04:22.172940812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51880"
ts=2026-05-18T12:04:27.722818535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:04:45.91204691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:04:52.174391221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58182"
ts=2026-05-18T12:04:57.724545692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:05:12.647580368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47750"
level=error ts=2026-05-18T12:05:22.530304569Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:05:22.532257836Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:05:22.534180333Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:05:22.534206282Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:05:27.72728351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:05:45.761463614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:05:57.729033565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:06:27.730335154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:06:42.654879766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:46626"
level=debug ts=2026-05-18T12:06:45.048043914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:06:52.184322251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:53264"
ts=2026-05-18T12:06:57.731715946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:07:25.195930888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54180"
ts=2026-05-18T12:07:27.733009487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:07:45.081488499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:07:57.734078661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:08:12.664295357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38616"
ts=2026-05-18T12:08:27.736410013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:08:36.704038579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:08:36.705579685Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:08:52.194950498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59326"
ts=2026-05-18T12:08:55.204171152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37486"
ts=2026-05-18T12:08:57.739023525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:09:12.670964631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47270"
level=debug ts=2026-05-18T12:09:13.352927226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:09:25.206787417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42138"
ts=2026-05-18T12:09:27.741951203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:09:57.744015471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:10:07.467199897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:10:25.211903298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34662"
ts=2026-05-18T12:10:27.746530764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:10:53.313568282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:10:57.749743483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:11:27.753269587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:11:33.899267934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:11:52.211555055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40908"
ts=2026-05-18T12:11:57.756470082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:12:12.67972652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51570"
ts=2026-05-18T12:12:27.759845739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:12:32.741746441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:12:32.743554835Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:12:52.216180377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58816"
ts=2026-05-18T12:12:57.762542162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:13:15.804833678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:13:22.218101911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:35524"
ts=2026-05-18T12:13:27.766153524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:13:42.687538782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58600"
ts=2026-05-18T12:13:57.769597782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:14:11.521660938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:14:27.771898078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:14:48.998247198Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:14:52.225408288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:57316"
ts=2026-05-18T12:14:55.236059968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47598"
ts=2026-05-18T12:14:57.772981641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:15:12.696048913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51194"
level=error ts=2026-05-18T12:15:22.529872061Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:15:22.531674845Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:15:22.533580401Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:15:22.533604735Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T12:15:27.468446817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:15:27.774937264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:15:52.22967053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:36396"
ts=2026-05-18T12:15:57.777471285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:16:08.977200172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:16:08.97862054Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:16:27.778439597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:16:46.728441097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:16:52.233878861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46056"
ts=2026-05-18T12:16:57.780840595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:17:22.235005798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:56078"
ts=2026-05-18T12:17:27.783576071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:17:42.710735188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60554"
level=debug ts=2026-05-18T12:17:48.399946993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:17:57.786369774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:18:12.713634979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:44704"
ts=2026-05-18T12:18:22.238723168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54038"
ts=2026-05-18T12:18:27.78920191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:18:42.716522009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58594"
level=debug ts=2026-05-18T12:18:44.263460892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:18:55.263259233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41834"
ts=2026-05-18T12:18:57.792455558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:19:22.243319217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:44410"
ts=2026-05-18T12:19:25.266591175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43992"
ts=2026-05-18T12:19:27.795831812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:19:42.720467868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57260"
level=debug ts=2026-05-18T12:19:43.041508812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:19:52.24472534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:50024"
ts=2026-05-18T12:19:57.798467288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:20:12.723195249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57096"
level=debug ts=2026-05-18T12:20:21.124241427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:20:21.125760375Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:20:27.80151429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:20:57.803351351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:21:02.839136184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:21:22.251954185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:44374"
ts=2026-05-18T12:21:27.805071397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:21:42.728271004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:54440"
level=debug ts=2026-05-18T12:21:51.692872383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:21:52.253275545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:37538"
ts=2026-05-18T12:21:57.807380172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:22:12.731723241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56060"
ts=2026-05-18T12:22:27.809949065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:22:28.715563701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:22:42.735131373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51918"
ts=2026-05-18T12:22:57.812389651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:23:21.357909814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:23:22.260989797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:48064"
ts=2026-05-18T12:23:27.815827942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:23:42.739472352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50808"
ts=2026-05-18T12:23:57.818535883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:24:11.932624921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:24:11.934215594Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:24:12.742520448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:58470"
ts=2026-05-18T12:24:22.264670597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38224"
ts=2026-05-18T12:24:27.820542915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:24:42.745870759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47282"
ts=2026-05-18T12:24:55.289388972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:55188"
ts=2026-05-18T12:24:57.821817264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:24:58.406509307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:25:22.531072767Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:25:22.5331605Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:25:22.535262521Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:25:22.535299483Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:25:27.823497116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:25:40.156531648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:25:42.752937674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51106"
ts=2026-05-18T12:25:57.825601084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:26:12.755832861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:43372"
ts=2026-05-18T12:26:22.275436988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:41046"
ts=2026-05-18T12:26:25.298777112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42378"
ts=2026-05-18T12:26:27.827489174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:26:32.17595566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:26:55.301691913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:59668"
ts=2026-05-18T12:26:57.82973542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:27:10.329104034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:27:12.762154649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:46972"
ts=2026-05-18T12:27:22.279222017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59690"
ts=2026-05-18T12:27:25.304486699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33154"
ts=2026-05-18T12:27:27.832567981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:27:42.765124029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55924"
level=debug ts=2026-05-18T12:27:46.362030561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:27:46.363551837Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:27:57.836123024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:28:12.768068697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47768"
ts=2026-05-18T12:28:27.839289042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:28:33.09788497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:28:52.28748023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:48598"
ts=2026-05-18T12:28:57.842790781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:29:10.274263202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:29:25.312844389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47500"
ts=2026-05-18T12:29:27.845125756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:29:42.775621144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:56638"
ts=2026-05-18T12:29:55.316193714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54574"
ts=2026-05-18T12:29:57.847098754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:29:59.501070003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:30:25.319327827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:42962"
ts=2026-05-18T12:30:27.849553656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:30:47.149816718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:30:57.851446575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:31:12.783236599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34956"
ts=2026-05-18T12:31:25.323815443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44088"
ts=2026-05-18T12:31:27.853608358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:31:43.734451508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:31:43.736215445Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:31:52.304394246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:44506"
ts=2026-05-18T12:31:55.327389806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60314"
ts=2026-05-18T12:31:57.856606243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:32:12.787693899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53352"
ts=2026-05-18T12:32:27.858593244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:32:32.712261612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:32:52.308937968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51782"
ts=2026-05-18T12:32:57.861312571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:33:22.310028045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:37232"
level=debug ts=2026-05-18T12:33:25.264617027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:33:25.334756429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52840"
ts=2026-05-18T12:33:27.864576591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:33:52.311311234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59540"
ts=2026-05-18T12:33:57.867804462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:34:07.550215225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:34:12.794750193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47876"
ts=2026-05-18T12:34:27.871107163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:34:57.873811784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:35:09.411756376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:35:12.801894249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:40154"
level=error ts=2026-05-18T12:35:22.530393958Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:35:22.532396622Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:35:22.5341799Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:35:22.534199838Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:35:27.875912221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:35:50.801676898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:35:50.803756393Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:35:57.877688196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:36:22.325039694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:45962"
ts=2026-05-18T12:36:27.880348608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:36:29.050406597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:36:57.883390524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:37:04.408455384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:37:27.885515737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:37:42.814933265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:43986"
ts=2026-05-18T12:37:55.357860825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45896"
level=debug ts=2026-05-18T12:37:55.887143484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:37:57.887407252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:38:27.889979469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:38:42.821052024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50296"
level=debug ts=2026-05-18T12:38:49.483423293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:38:55.363987436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43636"
ts=2026-05-18T12:38:57.892245656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:39:12.824450368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47570"
ts=2026-05-18T12:39:27.894599369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:39:47.127816138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:39:47.12971838Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:39:55.370025893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60236"
ts=2026-05-18T12:39:57.896396049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:40:22.347144427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:43578"
ts=2026-05-18T12:40:27.898556322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:40:42.831773545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:38298"
level=debug ts=2026-05-18T12:40:48.359036046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:40:55.375961516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37254"
ts=2026-05-18T12:40:57.901024448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:41:12.834870255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:49486"
ts=2026-05-18T12:41:25.379433093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38808"
ts=2026-05-18T12:41:27.903683933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:41:31.650443726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:41:55.382101786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35564"
ts=2026-05-18T12:41:57.906105639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:42:12.680040094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:42:12.841104039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57066"
ts=2026-05-18T12:42:25.385091071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50686"
ts=2026-05-18T12:42:27.907231923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:42:42.844400926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:54434"
ts=2026-05-18T12:42:52.361504704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:43138"
ts=2026-05-18T12:42:57.909855168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:43:13.771489312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:43:27.912864682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:43:42.848820652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36848"
ts=2026-05-18T12:43:57.916460352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:44:11.294124969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:44:11.295772246Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:44:12.851474564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42196"
ts=2026-05-18T12:44:27.91989131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:44:52.43134506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:44:57.923393153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:45:12.856044792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57568"
ts=2026-05-18T12:45:22.374807033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40372"
level=error ts=2026-05-18T12:45:22.530212856Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:45:22.532206028Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:45:22.534134393Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:45:22.534161691Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:45:27.926220763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:45:38.025546047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:45:42.859510759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53838"
ts=2026-05-18T12:45:55.400484342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:48016"
ts=2026-05-18T12:45:57.928716625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:46:19.924812128Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:46:27.931015289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:46:52.381663836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:42216"
ts=2026-05-18T12:46:57.934308499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:47:02.849930176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:47:27.936525003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:47:38.31874681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:47:38.320249197Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:47:55.409047927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:35168"
ts=2026-05-18T12:47:57.938859083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:48:22.390447135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49788"
ts=2026-05-18T12:48:25.412455428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:49174"
level=debug ts=2026-05-18T12:48:25.839744274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:48:27.941034147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:48:52.392131692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:49876"
ts=2026-05-18T12:48:55.416019008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37136"
ts=2026-05-18T12:48:57.942587362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:49:05.759103905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:49:22.393553529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:35170"
ts=2026-05-18T12:49:25.418979527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34564"
ts=2026-05-18T12:49:27.94405934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:49:55.422612622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37394"
ts=2026-05-18T12:49:57.946261372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:50:07.256075699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:50:22.398100945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:56784"
ts=2026-05-18T12:50:27.947951437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:50:55.426551511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51140"
ts=2026-05-18T12:50:57.949235625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:51:07.857378872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:51:22.402466321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59552"
ts=2026-05-18T12:51:25.430065918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51082"
ts=2026-05-18T12:51:27.950318977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:51:42.88389174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34724"
level=debug ts=2026-05-18T12:51:43.263041631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:51:43.264710042Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:51:57.952463111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T12:52:27.95615873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:52:33.688470936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:52:42.888125209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:41572"
ts=2026-05-18T12:52:57.958064292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:53:14.978508677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:53:22.412166317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:56076"
ts=2026-05-18T12:53:27.960872513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:53:55.441549773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37066"
ts=2026-05-18T12:53:57.963889802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:54:09.87456193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:54:22.416667191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:36876"
ts=2026-05-18T12:54:27.965862366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T12:54:57.966839531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:55:01.554989753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:55:22.4206973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55688"
level=error ts=2026-05-18T12:55:22.529886356Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:55:22.531843235Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:55:22.533709812Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T12:55:22.533732109Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:55:27.96803837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:55:50.284935986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:55:50.286520704Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:55:52.421584328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:38754"
ts=2026-05-18T12:55:57.969877478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:56:12.902433454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36092"
ts=2026-05-18T12:56:22.423113987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58946"
ts=2026-05-18T12:56:27.973085559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:56:34.851412495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:56:57.97646722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:57:27.978674816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T12:57:28.922935465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:57:42.909415562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:34888"
ts=2026-05-18T12:57:52.431061515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:48914"
ts=2026-05-18T12:57:57.981324285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:58:03.395352698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:58:12.91295305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51454"
ts=2026-05-18T12:58:22.432153113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:35736"
ts=2026-05-18T12:58:27.984203167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T12:58:42.915931646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:55650"
ts=2026-05-18T12:58:57.987476004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T12:59:00.251205028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T12:59:27.99099218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T12:59:35.462085625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T12:59:35.463890721Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T12:59:42.922589597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:48466"
ts=2026-05-18T12:59:57.993691518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:00:19.219388087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:00:27.997224303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:00:52.446751659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:55330"
level=debug ts=2026-05-18T13:00:54.835728408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:00:57.999892936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:01:12.93186993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52452"
ts=2026-05-18T13:01:25.475946172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:51286"
ts=2026-05-18T13:01:28.002585511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:01:37.866222764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:01:42.93521392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:33212"
ts=2026-05-18T13:01:52.450789595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:46432"
ts=2026-05-18T13:01:58.004566085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:02:22.452242474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:53492"
ts=2026-05-18T13:02:25.482180121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53074"
ts=2026-05-18T13:02:28.017973332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:02:34.253530756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:02:55.485281911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:49556"
ts=2026-05-18T13:02:58.020330151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:03:22.456352312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:45960"
ts=2026-05-18T13:03:28.022584637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:03:32.394948486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:03:32.396793505Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:03:42.943815388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60642"
ts=2026-05-18T13:03:52.457788696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:47286"
ts=2026-05-18T13:03:55.491619024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45114"
ts=2026-05-18T13:03:58.024670299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:04:22.458756651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54042"
level=debug ts=2026-05-18T13:04:23.227143744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:04:28.02710285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:04:42.950400258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50514"
ts=2026-05-18T13:04:55.497741518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60272"
ts=2026-05-18T13:04:58.029724703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:05:03.63951119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:05:22.530623492Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:05:22.532514156Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:05:22.534302068Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:05:22.534329031Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:05:28.03283931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:05:41.08795609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:05:58.036090178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:06:12.957824895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37474"
ts=2026-05-18T13:06:22.468857828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:36624"
level=debug ts=2026-05-18T13:06:27.517901221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:06:28.039757054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:06:42.961019434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59354"
ts=2026-05-18T13:06:58.042357557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:07:02.830179944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:07:02.831873608Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:07:28.044819169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:07:57.580021936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:07:58.047734508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:08:25.513482027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50826"
ts=2026-05-18T13:08:28.049975066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:08:32.440695748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:08:55.516547464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:37520"
ts=2026-05-18T13:08:58.052071955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:09:12.971577995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:60268"
level=debug ts=2026-05-18T13:09:16.988841867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:09:28.055679812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:09:42.974429399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59978"
level=debug ts=2026-05-18T13:09:55.625236759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:09:58.058161946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:10:22.49470107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:33820"
ts=2026-05-18T13:10:28.059499303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:10:42.384271445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:10:42.385741935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:10:42.97885728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:59826"
ts=2026-05-18T13:10:58.061890497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:11:12.981828918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:35238"
ts=2026-05-18T13:11:28.065443307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:11:29.679599494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:11:55.530617758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50836"
ts=2026-05-18T13:11:58.068243215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:12:28.071400168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:12:29.358245022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:12:42.989316386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:50888"
ts=2026-05-18T13:12:58.074483913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:13:11.461585587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:13:22.513588986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34542"
ts=2026-05-18T13:13:25.535552905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41518"
ts=2026-05-18T13:13:28.076572966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:13:42.99342941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:33268"
ts=2026-05-18T13:13:55.538777981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60430"
ts=2026-05-18T13:13:58.077694906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:14:07.061513517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:14:25.542015286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:44504"
ts=2026-05-18T13:14:28.078971301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:14:44.336789293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:14:44.338414027Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:14:52.521367024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:58710"
ts=2026-05-18T13:14:58.08175941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=error ts=2026-05-18T13:15:22.530464725Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:15:22.532368728Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:15:22.534292458Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:15:22.534315367Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-18T13:15:27.134044047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:15:28.083789939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:15:55.548377329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:49116"
ts=2026-05-18T13:15:58.085680248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:16:13.003329154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:53314"
level=debug ts=2026-05-18T13:16:14.661531933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:16:28.086872019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:16:55.554363507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:60228"
ts=2026-05-18T13:16:58.089076478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:17:07.906461478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:17:28.091087088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:17:58.09383202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:17:58.645790342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:18:22.540920953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:35306"
ts=2026-05-18T13:18:28.095834572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:18:43.017413207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:39742"
level=debug ts=2026-05-18T13:18:56.935607514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:18:56.93749373Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:18:58.09826841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:19:28.100781415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:19:58.102269481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:19:58.369619509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:20:25.571093982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:38502"
ts=2026-05-18T13:20:28.10469152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:20:36.181780842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:20:55.574620154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:46402"
ts=2026-05-18T13:20:58.107307254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:21:22.033476152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:21:28.110054041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:21:52.56059856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40982"
ts=2026-05-18T13:21:58.113205496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:22:13.03504756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:45974"
level=debug ts=2026-05-18T13:22:16.866121188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:22:28.116198641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:22:52.564638329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:51702"
ts=2026-05-18T13:22:55.581741859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53466"
ts=2026-05-18T13:22:58.118826263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:23:13.0419202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47212"
level=debug ts=2026-05-18T13:23:15.311017214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:23:15.312524809Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:23:25.584456139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:43602"
ts=2026-05-18T13:23:28.120578089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:23:43.045035523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36818"
level=debug ts=2026-05-18T13:23:50.634915321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:23:58.123834216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:24:22.57182599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:50500"
ts=2026-05-18T13:24:25.589766416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:50564"
ts=2026-05-18T13:24:28.126153741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:24:46.714958189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:24:58.128939773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=error ts=2026-05-18T13:25:22.530116199Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:25:22.532115459Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:25:22.534069031Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:25:22.534090502Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:25:25.595705256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:40518"
ts=2026-05-18T13:25:28.132392671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:25:33.483419417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:25:43.05361132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:41566"
ts=2026-05-18T13:25:52.578923565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:37572"
ts=2026-05-18T13:25:58.13552306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:26:25.599735249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45304"
ts=2026-05-18T13:26:28.138145367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:26:33.916697029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:26:55.602887251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53080"
ts=2026-05-18T13:26:58.139635904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:27:22.586718037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54278"
ts=2026-05-18T13:27:28.141945649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:27:34.686142897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:27:34.687589822Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:27:58.144698565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:28:13.068118814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:42230"
level=debug ts=2026-05-18T13:28:18.559122023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:28:28.147724074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:28:58.150385574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:29:03.859536199Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:29:28.152355142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:29:45.756450126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:29:58.154579967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:30:22.604111292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39208"
ts=2026-05-18T13:30:28.157998328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:30:30.575805731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:30:43.077725961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:46080"
ts=2026-05-18T13:30:58.161419822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:31:22.609442979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:52604"
ts=2026-05-18T13:31:25.624330639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54170"
ts=2026-05-18T13:31:28.164926314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:31:28.88748759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:31:28.889897383Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:31:43.081761772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:39972"
ts=2026-05-18T13:31:58.168568623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:32:13.084725617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:51996"
level=debug ts=2026-05-18T13:32:19.147064137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:32:28.171778269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:32:58.173683743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:33:11.619471552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:33:13.088125878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:37396"
ts=2026-05-18T13:33:22.620781655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:59234"
ts=2026-05-18T13:33:25.632941849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:33708"
ts=2026-05-18T13:33:28.176861358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:33:58.18017848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:34:08.483795689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:34:28.183486955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:34:58.186550032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:35:03.143323394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:35:22.530177954Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:35:22.532161032Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:35:22.534002709Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:35:22.534025013Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:35:28.189240823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:35:50.760696207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:35:50.762488667Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:35:58.192792993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:36:22.637623436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:43134"
ts=2026-05-18T13:36:28.195580821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:36:43.098774018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:44934"
level=debug ts=2026-05-18T13:36:47.353056538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:36:58.198187636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:37:22.641558614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:54558"
ts=2026-05-18T13:37:28.201257855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:37:43.105457581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47202"
level=debug ts=2026-05-18T13:37:43.474414514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:37:58.203248984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:38:21.423230009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:38:22.646200954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:34462"
ts=2026-05-18T13:38:28.2060256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:38:52.647541602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:35406"
ts=2026-05-18T13:38:58.208865069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:39:15.599905304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:39:25.662141947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52730"
ts=2026-05-18T13:39:28.211811882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
ts=2026-05-18T13:39:55.66520782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:53692"
level=debug ts=2026-05-18T13:39:56.945080389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:39:56.946967103Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:39:58.213671302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:40:13.116639293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:36582"
ts=2026-05-18T13:40:25.667759078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45940"
ts=2026-05-18T13:40:28.21584306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:40:41.618406862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:40:43.119371293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:47350"
ts=2026-05-18T13:40:55.670892579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:41694"
ts=2026-05-18T13:40:58.218958377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:41:13.122562799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52572"
ts=2026-05-18T13:41:28.222303235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:41:33.959798197Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:41:52.665217059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:39240"
ts=2026-05-18T13:41:55.677525115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:34432"
ts=2026-05-18T13:41:58.224353165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:42:25.680927962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:36060"
ts=2026-05-18T13:42:28.226022277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:42:30.749847183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:42:43.12954375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:57890"
ts=2026-05-18T13:42:55.683877355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:39060"
ts=2026-05-18T13:42:58.227018877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
level=debug ts=2026-05-18T13:43:07.73728456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:43:22.673251234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:33940"
ts=2026-05-18T13:43:28.228110455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:43:52.674202996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:42388"
ts=2026-05-18T13:43:55.688570878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:57960"
ts=2026-05-18T13:43:58.230732317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:44:04.808303079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:44:04.809936124Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:44:13.13533415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:52726"
ts=2026-05-18T13:44:28.232630387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:44:52.678589515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:40860"
ts=2026-05-18T13:44:58.235227924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
level=debug ts=2026-05-18T13:45:07.12486357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-18T13:45:22.530042197Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:45:22.531758665Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:45:22.533445528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-18T13:45:22.53346986Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-18T13:45:25.69771058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:54944"
ts=2026-05-18T13:45:28.237817951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54ef3698 10.129.0.77:7946"
ts=2026-05-18T13:45:58.239954344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-8jx89-18527382 10.130.0.73:7946"
level=debug ts=2026-05-18T13:46:06.960598242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-18T13:46:13.144485278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.73:48804"
ts=2026-05-18T13:46:28.243186877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-bznh9-aa15549e 10.130.0.72:7946"
ts=2026-05-18T13:46:52.689519678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.77:43754"