level=info ts=2026-05-09T18:35:10.318751863Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-09T18:35:10.318788661Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-09T18:35:10.321692666Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-09T18:35:10.322135912Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T18:35:10.32224141Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.66 inf=eth0
level=debug ts=2026-05-09T18:35:10.322935659Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T18:35:10.323022424Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.66 inf=eth0
level=info ts=2026-05-09T18:35:10.323126722Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-fb1e97e6
ts=2026-05-09T18:35:10.323199475Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-09T18:35:10.323210992Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.66 advertisePort=7946
level=debug ts=2026-05-09T18:35:10.323225883Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.66 advertisePort=7946
level=info ts=2026-05-09T18:35:10.323339232Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-09T18:35:10.330125158Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T18:35:10.330146068Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T18:35:10.330155793Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.818671ms
level=error ts=2026-05-09T18:35:10.330165177Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-09T18:35:10.330176443Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-09T18:35:10.332527033Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T18:35:10.332541695Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-09T18:35:10.339977574Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-09T18:35:10.339988954Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-09T18:35:10.339972551Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-09T18:35:10.339996918Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-09T18:35:10.340023261Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-09T18:35:10.340047985Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-09T18:35:10.3401102Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-09T18:35:10.340125976Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-09T18:35:10.340338256Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-09T18:35:10.34035091Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-09T18:35:10.340476909Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-09T18:35:10.340499107Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-09T18:35:10.340527492Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-09T18:35:10.340704057Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-09T18:35:10.340720106Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-09T18:35:10.340751536Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-09T18:35:11.341490909Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-09T18:35:11.341530437Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-09T18:35:11.34154886Z caller=loki.go:615 msg="Loki started" startup_time=1.08059404s
ts=2026-05-09T18:35:11.721953658Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T18:35:11.721982842Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:35:15.152505353Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T18:35:15.152533762Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T18:35:16.341816958Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-09T18:35:16.341878835Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-09T18:35:21.536763472Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
ts=2026-05-09T18:35:21.538182716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.75:7946"
ts=2026-05-09T18:35:21.541243672Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.83:7946: dial tcp 10.128.0.83:7946: connect: connection refused"
ts=2026-05-09T18:35:21.542635607Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.81:7946: dial tcp 10.128.0.81:7946: connect: connection refused"
ts=2026-05-09T18:35:21.544040548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.82:7946"
level=info ts=2026-05-09T18:35:21.545264951Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=11.21508385s
level=debug ts=2026-05-09T18:35:31.249954494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:35:32.264491013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57500"
ts=2026-05-09T18:35:45.658842553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:35:49.072692238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:35:57.764169545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39174"
level=debug ts=2026-05-09T18:36:13.720365591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:36:15.661474266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:36:37.636919504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:36:45.664005439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:36:57.76976924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45122"
level=debug ts=2026-05-09T18:37:02.130191519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T18:37:02.132170585Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T18:37:15.667092137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:37:27.772326503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:34882"
ts=2026-05-09T18:37:39.311458336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:32906"
level=debug ts=2026-05-09T18:37:42.692666214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:37:45.669906855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:37:58.582818855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53612"
ts=2026-05-09T18:38:09.31438741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52708"
ts=2026-05-09T18:38:15.672569684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:38:28.586128075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44238"
ts=2026-05-09T18:38:45.675352111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T18:39:03.897889561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:39:15.678281793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:39:45.680891836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T18:39:58.592688911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44976"
level=debug ts=2026-05-09T18:40:02.947354583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:40:15.68421498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:40:45.687016601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:40:58.597323743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:44878"
level=debug ts=2026-05-09T18:40:59.230789232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:41:09.324734878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59874"
ts=2026-05-09T18:41:15.689819659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T18:41:28.600670875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59280"
ts=2026-05-09T18:41:45.693204677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T18:42:15.337862732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T18:42:15.33947934Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T18:42:15.696536742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T18:42:28.605021569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51386"
ts=2026-05-09T18:42:45.699368184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T18:43:07.011563003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:43:15.702828032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:43:28.610438976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36554"
ts=2026-05-09T18:43:45.705360795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:43:49.390431606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:44:15.707891899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:44:27.169078736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:44:39.339596323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60910"
ts=2026-05-09T18:44:45.711279243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:44:58.616588434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:52166"
level=debug ts=2026-05-09T18:45:07.591698703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:45:15.714279924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=info ts=2026-05-09T18:45:16.342740979Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-09T18:45:16.345171773Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T18:45:16.34746879Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T18:45:16.349240925Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T18:45:16.349279459Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T18:45:16.349292635Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-09T18:45:27.806509612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51568"
level=debug ts=2026-05-09T18:45:43.774124792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T18:45:43.77557825Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T18:45:45.717731752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:45:57.809106298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:34956"
ts=2026-05-09T18:46:09.345570461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55072"
ts=2026-05-09T18:46:15.721189951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:46:37.216298224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:46:39.349066893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35238"
ts=2026-05-09T18:46:45.724587328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:47:15.728027272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T18:47:27.816250961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42134"
ts=2026-05-09T18:47:28.626162938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56460"
level=debug ts=2026-05-09T18:47:34.781362978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:47:39.353581026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55390"
ts=2026-05-09T18:47:45.730724101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:48:09.356556767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38646"
ts=2026-05-09T18:48:15.733537794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T18:48:25.735548502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:48:45.737106899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:49:09.361639511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:32960"
level=debug ts=2026-05-09T18:49:09.827017319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:49:15.74028628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:49:28.635441037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40448"
ts=2026-05-09T18:49:45.743061418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:49:57.826889336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38446"
level=debug ts=2026-05-09T18:49:58.422210616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T18:49:58.424078679Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T18:49:58.638456044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48176"
ts=2026-05-09T18:50:15.746130879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:50:45.749557689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:50:56.494449178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:50:57.832018002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59258"
ts=2026-05-09T18:51:15.752742653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T18:51:27.835232881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42082"
ts=2026-05-09T18:51:28.644365367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59822"
ts=2026-05-09T18:51:45.756083209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:51:47.015905439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:51:57.838203234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49518"
ts=2026-05-09T18:52:15.759116813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:52:27.840810456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43350"
ts=2026-05-09T18:52:28.649998774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48768"
ts=2026-05-09T18:52:39.374841261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37306"
level=debug ts=2026-05-09T18:52:40.147820981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:52:45.761633065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T18:52:57.843840106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37664"
ts=2026-05-09T18:53:15.764851941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T18:53:17.709881317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:53:45.767717993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:53:56.052524385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T18:53:56.05469774Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T18:53:58.656497516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41270"
ts=2026-05-09T18:54:15.770670802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:54:27.851706045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46244"
ts=2026-05-09T18:54:39.384773853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59444"
level=debug ts=2026-05-09T18:54:41.573281516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:54:45.774213208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T18:54:57.855085812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58752"
ts=2026-05-09T18:55:09.388072592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52992"
ts=2026-05-09T18:55:15.776978305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=error ts=2026-05-09T18:55:16.352832975Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T18:55:16.355035608Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T18:55:16.357208446Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T18:55:16.357231486Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T18:55:18.053335258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:55:28.663180153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43418"
ts=2026-05-09T18:55:39.391601389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33260"
ts=2026-05-09T18:55:45.780599794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T18:56:13.811073758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:56:15.784328012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:56:39.395648154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45896"
ts=2026-05-09T18:56:45.787274683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T18:57:10.282682996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:57:15.790600234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T18:57:28.670388058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45742"
ts=2026-05-09T18:57:45.794147928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T18:57:54.138269744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T18:57:54.140185105Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T18:58:15.800082062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T18:58:45.803144742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:58:56.408956714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T18:59:15.806285731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T18:59:45.809191195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T18:59:54.418117544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:00:09.413887652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60434"
ts=2026-05-09T19:00:15.812557542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:00:27.878743216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:56892"
ts=2026-05-09T19:00:39.417569153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35660"
ts=2026-05-09T19:00:45.823650121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:00:53.188692177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:00:57.882094802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54412"
ts=2026-05-09T19:01:15.830337534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:01:29.080145904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:01:45.833809498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:01:57.887488079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59188"
ts=2026-05-09T19:02:09.426960343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56900"
ts=2026-05-09T19:02:15.837352545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:02:17.175379649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:02:17.177087387Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:02:27.89078673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45188"
ts=2026-05-09T19:02:45.840356289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:03:02.870043253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:03:15.843691126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:03:27.896756908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:57090"
level=debug ts=2026-05-09T19:03:43.678196631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:03:45.847178422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:04:15.850669327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:04:28.696372975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49450"
level=debug ts=2026-05-09T19:04:45.548097798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:04:45.853432311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:05:09.438550331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48048"
ts=2026-05-09T19:05:15.856268514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=error ts=2026-05-09T19:05:16.354320211Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:05:16.356368399Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:05:16.35820225Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:05:16.358231925Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:05:28.70275873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59892"
ts=2026-05-09T19:05:39.442593787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33700"
level=debug ts=2026-05-09T19:05:44.614388811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:05:45.859259328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:05:57.90950633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38982"
ts=2026-05-09T19:06:15.862688554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:06:24.896555645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:06:24.898336586Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:06:45.865818683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:06:58.711441783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49818"
level=debug ts=2026-05-09T19:07:15.715105924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:07:15.868948857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:07:45.871656481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:07:49.846031355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:07:57.922751866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:57558"
ts=2026-05-09T19:07:58.716661172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56380"
ts=2026-05-09T19:08:15.874354861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:08:26.813354217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:08:45.877727952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:09:09.457505983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58874"
ts=2026-05-09T19:09:15.880456887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:09:26.630485279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:09:45.883279904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:10:03.239464299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:10:03.241231279Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:10:09.462166764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59426"
ts=2026-05-09T19:10:15.886649536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:10:27.93312986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:60756"
ts=2026-05-09T19:10:28.727638652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:33920"
ts=2026-05-09T19:10:45.890187697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:10:46.87134436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:11:09.467653548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54666"
ts=2026-05-09T19:11:15.893224209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:11:27.937845244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:50204"
ts=2026-05-09T19:11:28.73244218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:47480"
level=debug ts=2026-05-09T19:11:33.340706692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:11:45.896678877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:11:57.940315132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47500"
ts=2026-05-09T19:11:58.735585264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38912"
ts=2026-05-09T19:12:09.472730276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38076"
ts=2026-05-09T19:12:15.899896998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:12:15.924001467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:12:45.903242646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:12:57.945473013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45002"
level=debug ts=2026-05-09T19:13:09.135627261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:13:09.476603383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:33140"
ts=2026-05-09T19:13:15.906330651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:13:45.909227125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:13:52.205365355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:13:52.206992488Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:14:15.912312431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:14:27.953429244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53020"
level=debug ts=2026-05-09T19:14:38.047132356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:14:45.915638407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:14:57.956598396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:52708"
level=debug ts=2026-05-09T19:15:14.06253183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:15:15.91859137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=error ts=2026-05-09T19:15:16.351921328Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:15:16.353607388Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:15:16.355216543Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:15:16.355241873Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:15:27.959169416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:48376"
ts=2026-05-09T19:15:39.487526552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35506"
ts=2026-05-09T19:15:45.921474391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:15:57.962014706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54738"
level=debug ts=2026-05-09T19:16:13.005078028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:16:15.925148511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:16:27.964529852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42472"
ts=2026-05-09T19:16:28.751921302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46342"
ts=2026-05-09T19:16:39.492466752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48482"
ts=2026-05-09T19:16:45.928321752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:16:49.479449161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:16:57.967821153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:38740"
ts=2026-05-09T19:17:09.495367976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35968"
ts=2026-05-09T19:17:15.930521814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:17:27.970571193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:57046"
ts=2026-05-09T19:17:45.933046876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:17:46.195386352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:17:46.196913661Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:18:15.935769113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:18:27.975279554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46740"
level=debug ts=2026-05-09T19:18:28.245889254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:18:39.506040242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46508"
ts=2026-05-09T19:18:45.938940143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:19:07.584887431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:19:15.942064178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:19:39.512330447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36072"
ts=2026-05-09T19:19:45.944833167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:19:57.983760104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:42040"
level=debug ts=2026-05-09T19:20:03.370437913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:20:09.541217064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52296"
ts=2026-05-09T19:20:15.948106969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:20:45.951458932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:21:04.424174897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:21:15.954666803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:21:45.957802541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:21:56.540544283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:21:56.542552463Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:22:15.960749284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:22:39.548029844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:22:45.964073994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:22:57.999090095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47032"
ts=2026-05-09T19:23:09.552675026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52686"
level=debug ts=2026-05-09T19:23:15.483395858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:23:15.966599623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:23:28.77758907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48956"
ts=2026-05-09T19:23:45.970069128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:24:09.561361299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60724"
level=debug ts=2026-05-09T19:24:10.538528775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:24:15.973651453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:24:28.006447836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37396"
ts=2026-05-09T19:24:45.976976278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:24:47.324967721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:24:58.009609533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36944"
ts=2026-05-09T19:25:15.979455342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=error ts=2026-05-09T19:25:16.353944237Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:25:16.35610283Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:25:16.357873349Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:25:16.357896775Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T19:25:26.371898279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:25:26.374127007Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:25:39.568297208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45442"
ts=2026-05-09T19:25:45.982953234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:25:58.788124081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39502"
ts=2026-05-09T19:26:09.571180109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43406"
ts=2026-05-09T19:26:15.985940891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:26:22.533234084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:26:28.016473175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54540"
ts=2026-05-09T19:26:39.574230973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52274"
ts=2026-05-09T19:26:45.98894622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:26:58.792354504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:60924"
ts=2026-05-09T19:27:15.992527136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:27:21.134525799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:27:28.795431719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48038"
ts=2026-05-09T19:27:39.578824283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47966"
ts=2026-05-09T19:27:45.996048734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:28:09.581433159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52144"
level=debug ts=2026-05-09T19:28:12.555795446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:28:15.999111507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:28:28.800065339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40200"
ts=2026-05-09T19:28:46.00233147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:29:03.309571838Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:29:16.00498278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:29:38.141546939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:29:38.144421353Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:29:46.008028422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:30:16.010467061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:30:23.379029367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:30:28.032950674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55294"
ts=2026-05-09T19:30:28.809524516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55000"
ts=2026-05-09T19:30:46.013361057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:30:58.035449085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:59750"
ts=2026-05-09T19:30:58.812253991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53324"
level=debug ts=2026-05-09T19:31:08.020052865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:31:16.016805304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:31:39.595583778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59578"
ts=2026-05-09T19:31:46.020173652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:31:58.816194941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50844"
level=debug ts=2026-05-09T19:32:01.202790802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:32:09.598320064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58050"
ts=2026-05-09T19:32:16.022854504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:32:28.04244807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:52690"
ts=2026-05-09T19:32:28.819421492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34516"
ts=2026-05-09T19:32:46.025381031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:32:58.822566007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36286"
level=debug ts=2026-05-09T19:32:59.390970902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:33:09.604396779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54924"
ts=2026-05-09T19:33:16.027770195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:33:28.04755678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58594"
ts=2026-05-09T19:33:46.030885038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:33:57.824802101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:33:57.826545385Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:33:58.828366386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50668"
ts=2026-05-09T19:34:16.033637707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:34:28.052041885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:37094"
ts=2026-05-09T19:34:46.036705646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:34:47.150455386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:35:09.613655446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41876"
ts=2026-05-09T19:35:16.03928062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=error ts=2026-05-09T19:35:16.353751375Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:35:16.355653184Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:35:16.35771877Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:35:16.357740914Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:35:28.835577511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34134"
level=debug ts=2026-05-09T19:35:32.710396728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:35:46.042196311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:36:16.044641941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:36:17.147307305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:36:28.060320159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:47088"
ts=2026-05-09T19:36:39.621118539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58582"
ts=2026-05-09T19:36:46.047463524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:36:58.062896325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:56204"
ts=2026-05-09T19:36:58.842990664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48920"
level=debug ts=2026-05-09T19:37:06.344951466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:37:16.050568437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:37:28.065596976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:35036"
ts=2026-05-09T19:37:39.625271705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48000"
ts=2026-05-09T19:37:46.053208507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:37:52.810905937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:37:52.812374134Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:38:16.056562902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:38:39.628712812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35528"
level=debug ts=2026-05-09T19:38:44.385773215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:38:46.059182994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:39:16.061539252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:39:31.201881903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:39:46.06458205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:40:16.067788706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:40:27.245590303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:40:28.079552922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:45832"
ts=2026-05-09T19:40:46.07032052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:40:58.082334555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:53016"
ts=2026-05-09T19:40:58.861034804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36208"
ts=2026-05-09T19:41:16.073292019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:41:25.480755707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:41:28.864027849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53198"
ts=2026-05-09T19:41:39.641130156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39936"
ts=2026-05-09T19:41:46.076377354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:42:09.64467594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59866"
ts=2026-05-09T19:42:16.079541757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:42:19.614062573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:42:19.615433245Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:42:46.082680025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:43:15.628074246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:43:16.085609927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:43:28.093908747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:50320"
ts=2026-05-09T19:43:39.650802892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38004"
ts=2026-05-09T19:43:46.088801363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:43:51.858584333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:44:16.092155845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:44:28.876525375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46744"
ts=2026-05-09T19:44:46.095683667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:44:46.438752121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:44:58.879966825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50244"
ts=2026-05-09T19:45:09.65668666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36062"
ts=2026-05-09T19:45:16.098758216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=error ts=2026-05-09T19:45:16.355445528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:45:16.357266477Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:45:16.358917668Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:45:16.358939371Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T19:45:27.345218601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:45:28.882677402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42002"
ts=2026-05-09T19:45:46.101920332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:46:09.660510036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58706"
level=debug ts=2026-05-09T19:46:14.586617937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:46:14.588171437Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:46:16.104913512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:46:46.107504843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:46:57.396326003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:47:09.66475313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57714"
ts=2026-05-09T19:47:16.110523484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:47:32.436947545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:47:39.66784306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52488"
ts=2026-05-09T19:47:46.11349955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:48:14.256898204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:48:16.116817951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:48:46.119794773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:48:54.953618337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:48:58.898748213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:34204"
ts=2026-05-09T19:49:16.123561909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:49:39.675507281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42986"
ts=2026-05-09T19:49:46.126805093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:49:48.107890901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:49:48.109667116Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:50:16.12998642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:50:28.177654658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54830"
level=debug ts=2026-05-09T19:50:30.23614674Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:50:46.133472309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:50:58.180609894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:36548"
level=debug ts=2026-05-09T19:51:07.466308696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:51:16.136875442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:51:28.183209603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33890"
ts=2026-05-09T19:51:46.140455692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:51:51.698657636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:51:58.186465436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49994"
ts=2026-05-09T19:52:16.143465385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T19:52:28.91337919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49060"
level=debug ts=2026-05-09T19:52:34.859909361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:52:39.68838372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52280"
ts=2026-05-09T19:52:46.146024737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:53:09.691733015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43702"
ts=2026-05-09T19:53:16.149034281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:53:28.192438603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:46096"
level=debug ts=2026-05-09T19:53:32.606945994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:53:32.608612967Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:53:39.695272957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39724"
ts=2026-05-09T19:53:46.151567851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:53:58.919340608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51272"
ts=2026-05-09T19:54:16.154458562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:54:22.267161649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:54:28.922888788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48082"
ts=2026-05-09T19:54:46.157208411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:55:08.062661729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:55:09.700927141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57358"
ts=2026-05-09T19:55:16.15964581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=error ts=2026-05-09T19:55:16.352961864Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:55:16.354738257Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:55:16.356541411Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T19:55:16.356564174Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:55:28.201109891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:43376"
ts=2026-05-09T19:55:46.161997479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:55:59.562209767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:56:09.706715306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46872"
ts=2026-05-09T19:56:16.164628054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T19:56:43.709710912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:56:46.16812226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:57:16.17132675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T19:57:28.659520844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:57:28.661117354Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:57:46.173997156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T19:57:58.212905578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33398"
ts=2026-05-09T19:57:58.936680336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37122"
ts=2026-05-09T19:58:09.715094066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58170"
ts=2026-05-09T19:58:16.177346819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:58:25.993998316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:58:28.216237927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:56044"
ts=2026-05-09T19:58:28.940016088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:57838"
ts=2026-05-09T19:58:46.180705646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:58:58.219284719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:58534"
ts=2026-05-09T19:59:09.719951528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39566"
ts=2026-05-09T19:59:16.183585607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T19:59:19.301572009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:59:28.221892859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:41724"
ts=2026-05-09T19:59:28.945557764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:51510"
ts=2026-05-09T19:59:46.186597131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T19:59:58.224866241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54662"
ts=2026-05-09T19:59:58.949011899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39520"
level=debug ts=2026-05-09T20:00:00.161564596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:00:16.1899106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T20:00:46.192537158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T20:00:48.36256001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:01:16.195690838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T20:01:28.955109633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39066"
level=debug ts=2026-05-09T20:01:31.877904037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:01:31.879775641Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T20:01:39.732724087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48866"
ts=2026-05-09T20:01:46.198393851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T20:02:08.962541592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:02:16.201242669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T20:02:28.959992214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39490"
ts=2026-05-09T20:02:46.204325824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T20:03:01.551944162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:03:16.207116857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T20:03:37.540621247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:03:46.209838741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T20:03:58.966443179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:48444"
ts=2026-05-09T20:04:09.742725007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46482"
level=debug ts=2026-05-09T20:04:14.279005081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:04:16.212843805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T20:04:28.244197786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:55382"
ts=2026-05-09T20:04:39.74550489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52840"
ts=2026-05-09T20:04:46.215637314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T20:04:58.97172908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37650"
level=debug ts=2026-05-09T20:05:13.25477753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:05:13.256315631Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T20:05:16.218595155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=error ts=2026-05-09T20:05:16.351624641Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T20:05:16.353267812Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T20:05:16.354713471Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T20:05:16.35473236Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T20:05:28.9745709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:43200"
ts=2026-05-09T20:05:39.750636406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57078"
ts=2026-05-09T20:05:46.221828779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T20:06:07.994759724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:06:09.754011407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48850"
ts=2026-05-09T20:06:16.224273098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T20:06:28.251861574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39158"
ts=2026-05-09T20:06:39.756795824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43044"
ts=2026-05-09T20:06:46.226954516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T20:06:53.187577337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:06:58.25544497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:39672"
ts=2026-05-09T20:07:09.760212794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43474"
ts=2026-05-09T20:07:16.229878957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T20:07:33.331818261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:07:46.232602237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T20:07:58.260003853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:51958"
ts=2026-05-09T20:07:58.985214379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:54922"
ts=2026-05-09T20:08:09.76566738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60060"
level=debug ts=2026-05-09T20:08:11.09029909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:08:16.235230024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T20:08:39.768736311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35568"
ts=2026-05-09T20:08:46.238278123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T20:08:57.42319375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:08:57.425003356Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T20:09:09.771992616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:55182"
ts=2026-05-09T20:09:16.241513466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
ts=2026-05-09T20:09:28.992567338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40080"
ts=2026-05-09T20:09:46.244877623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T20:09:52.242444207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:09:58.268579423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:54514"
ts=2026-05-09T20:10:16.247566758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T20:10:27.183221193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:10:46.250808364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T20:10:58.273098468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:33018"
ts=2026-05-09T20:11:16.253478313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
level=debug ts=2026-05-09T20:11:20.61240029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:11:39.782028387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51574"
ts=2026-05-09T20:11:46.256050962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T20:11:58.278142304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49478"
ts=2026-05-09T20:11:59.005067043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:41710"
level=debug ts=2026-05-09T20:12:01.521039889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:12:09.785646332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38614"
ts=2026-05-09T20:12:16.258867305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T20:12:29.008173222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46146"
ts=2026-05-09T20:12:39.789169552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50912"
ts=2026-05-09T20:12:46.261925562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T20:12:54.868947287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:12:54.870583182Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T20:12:58.283121712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:49386"
ts=2026-05-09T20:13:09.792503374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36820"
ts=2026-05-09T20:13:16.26494896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ab999151 10.129.0.78:7946"
ts=2026-05-09T20:13:39.796013235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46674"
ts=2026-05-09T20:13:46.26849149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T20:13:52.759487114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:14:09.798606793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53944"
ts=2026-05-09T20:14:16.271566597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T20:14:46.274352617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=debug ts=2026-05-09T20:14:50.428615222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:14:58.291976238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.82:40620"
ts=2026-05-09T20:15:16.277232649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfq5c-963acb03 10.128.0.82:7946"
level=error ts=2026-05-09T20:15:16.355377076Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T20:15:16.35757067Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T20:15:16.359209944Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-09T20:15:16.359234035Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T20:15:39.804597557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41030"
level=debug ts=2026-05-09T20:15:41.811618842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:15:46.280272414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
ts=2026-05-09T20:15:59.020936941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40420"
ts=2026-05-09T20:16:16.283398027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"
level=debug ts=2026-05-09T20:16:26.92006346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:16:29.024640902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49120"
ts=2026-05-09T20:16:46.286210854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xhcm5-8ee8bfb5 10.129.0.75:7946"