level=info ts=2026-05-12T21:23:14.061345031Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-12T21:23:14.061384362Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-12T21:23:14.063553778Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-12T21:23:14.064012383Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-12T21:23:14.064172449Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.73 inf=eth0
level=debug ts=2026-05-12T21:23:14.064811662Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=info ts=2026-05-12T21:23:14.06489486Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-84031014
level=debug ts=2026-05-12T21:23:14.064901952Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.73 inf=eth0
ts=2026-05-12T21:23:14.064953325Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-12T21:23:14.064977149Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.73 advertisePort=7946
level=debug ts=2026-05-12T21:23:14.064994368Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.73 advertisePort=7946
level=info ts=2026-05-12T21:23:14.06511914Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-12T21:23:14.074519764Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T21:23:14.07454268Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-12T21:23:14.074557253Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=9.43926ms
level=error ts=2026-05-12T21:23:14.074569679Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-12T21:23:14.074580382Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-12T21:23:14.078267058Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:23:14.078294532Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:23:14.083333441Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-12T21:23:14.083387602Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-12T21:23:14.083403274Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-12T21:23:14.083397153Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-12T21:23:14.08339338Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-12T21:23:14.083445494Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-12T21:23:14.083542307Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-12T21:23:14.083557521Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-12T21:23:14.083661677Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-12T21:23:14.08367015Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-12T21:23:14.083720652Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-12T21:23:14.083758734Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-12T21:23:14.08376739Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-12T21:23:14.083904924Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-12T21:23:14.083914508Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-12T21:23:14.083929443Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-12T21:23:15.084899314Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
ts=2026-05-12T21:23:15.098495871Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:23:15.098518865Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-12T21:23:15.224169742Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-12T21:23:15.224235921Z caller=loki.go:615 msg="Loki started" startup_time=1.227723038s
ts=2026-05-12T21:23:17.168272859Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:23:17.168301779Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-12T21:23:20.225084852Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-12T21:23:20.225134321Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-12T21:23:21.370335102Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-12T21:23:21.370359331Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-12T21:23:29.98325688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:23:34.265062177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.73:7946"
ts=2026-05-12T21:23:34.265183743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:35502"
ts=2026-05-12T21:23:34.267742763Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.61:7946: dial tcp 10.128.0.61:7946: connect: connection refused"
level=info ts=2026-05-12T21:23:34.267757128Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=20.193170184s
ts=2026-05-12T21:23:34.821586282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34274"
ts=2026-05-12T21:23:37.223522958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49580"
ts=2026-05-12T21:23:37.375646131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52546"
ts=2026-05-12T21:23:44.771884569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:23:45.471632743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:23:54.499911174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45966"
level=debug ts=2026-05-12T21:24:08.047922219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:24:14.774774369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:24:24.503634684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:39632"
level=debug ts=2026-05-12T21:24:26.567081153Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:24:44.777681078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:24:54.507053953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:59868"
level=debug ts=2026-05-12T21:25:01.929250416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T21:25:01.931219207Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:25:14.781284461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:25:35.069365965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60282"
ts=2026-05-12T21:25:44.783460459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:25:48.486999438Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:25:54.513179719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:33882"
ts=2026-05-12T21:26:05.070403487Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44804"
ts=2026-05-12T21:26:14.785818604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:26:24.516505229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:59664"
ts=2026-05-12T21:26:35.071824686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56156"
ts=2026-05-12T21:26:41.1473484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45600"
level=debug ts=2026-05-12T21:26:44.416627255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:26:44.787775492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:26:54.519410191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:39794"
ts=2026-05-12T21:27:05.073148747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58412"
ts=2026-05-12T21:27:14.789537951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:27:35.074016539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39876"
level=debug ts=2026-05-12T21:27:36.517748817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:27:41.150063969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55674"
ts=2026-05-12T21:27:44.790652891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:28:05.074730919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36284"
ts=2026-05-12T21:28:14.791621773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:28:35.076207489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36898"
ts=2026-05-12T21:28:44.793273339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:28:51.232225322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-12T21:29:14.795305782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:29:35.078835236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44700"
level=debug ts=2026-05-12T21:29:44.148191475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-12T21:29:44.14967721Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:29:44.797575214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:30:14.799986813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:30:27.951213984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:30:44.802652258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:30:54.543618235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53566"
level=debug ts=2026-05-12T21:31:06.766628315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:31:14.804418787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:31:24.54636231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48136"
ts=2026-05-12T21:31:35.084082462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33804"
ts=2026-05-12T21:31:41.164310437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35592"
ts=2026-05-12T21:31:44.805797464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:31:45.643973994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:31:54.549500927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:51222"
ts=2026-05-12T21:32:05.085321458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51346"
ts=2026-05-12T21:32:14.807224319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:32:26.359082218Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:32:44.80919959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:33:11.171541483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46132"
ts=2026-05-12T21:33:14.811002702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=info ts=2026-05-12T21:33:20.225937705Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-12T21:33:20.232492Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:33:20.23540046Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:33:20.238329672Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:33:20.238370171Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-12T21:33:20.238389438Z caller=tables_manager.go:148 msg="compactor started"
level=debug ts=2026-05-12T21:33:26.52691291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:33:26.529411805Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:33:44.812189491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T21:34:05.094635652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40596"
ts=2026-05-12T21:34:14.813619019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:34:21.262280842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:34:35.095604278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60398"
ts=2026-05-12T21:34:41.178441771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44118"
ts=2026-05-12T21:34:44.81684662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:34:54.568685437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48054"
level=debug ts=2026-05-12T21:35:10.22543077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:35:14.820781869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:35:24.572034772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40520"
ts=2026-05-12T21:35:44.824768517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:35:54.575929629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:33764"
level=debug ts=2026-05-12T21:36:05.138210266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:36:14.826988445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:36:44.828261652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:37:02.938279319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:37:05.107359055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41346"
ts=2026-05-12T21:37:11.191957563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52740"
ts=2026-05-12T21:37:14.831360524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:37:44.833163421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:38:01.632614843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:38:01.634749759Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:38:14.834728969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:38:24.592852697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:37596"
ts=2026-05-12T21:38:35.114524037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53894"
ts=2026-05-12T21:38:44.837768165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:39:00.866910725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:39:14.84052571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:39:41.211130954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48956"
level=debug ts=2026-05-12T21:39:44.783793465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:39:44.842999933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:39:54.604293305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45404"
ts=2026-05-12T21:40:05.121485874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:57554"
ts=2026-05-12T21:40:11.212753583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44228"
ts=2026-05-12T21:40:14.844744529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:40:27.726225343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:40:35.122910263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60260"
ts=2026-05-12T21:40:41.21402623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39516"
ts=2026-05-12T21:40:44.846199642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:41:05.566937286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:41:11.215660173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:37418"
ts=2026-05-12T21:41:14.847532378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:41:41.217624461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51242"
ts=2026-05-12T21:41:44.849384829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:41:48.218416525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:41:48.220110492Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:41:54.61750941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:51674"
ts=2026-05-12T21:42:05.126198058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54502"
ts=2026-05-12T21:42:14.851307242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:42:35.127506797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50418"
level=debug ts=2026-05-12T21:42:36.406240774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:42:44.854105264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:43:11.224231981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46538"
level=debug ts=2026-05-12T21:43:14.203990282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:43:14.855649426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=error ts=2026-05-12T21:43:20.241569038Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:43:20.243548661Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:43:20.245333039Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:43:20.245353349Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:43:24.628815062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:42252"
ts=2026-05-12T21:43:35.130651473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54882"
ts=2026-05-12T21:43:41.22539394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40142"
ts=2026-05-12T21:43:44.857211882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:43:54.631631241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:56938"
level=debug ts=2026-05-12T21:43:56.49002296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:44:14.859322191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:44:35.136061814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44360"
ts=2026-05-12T21:44:44.861674566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:44:50.801658758Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:44:54.638249734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:39898"
ts=2026-05-12T21:45:14.863608303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:45:24.641704638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:60232"
level=debug ts=2026-05-12T21:45:26.448181994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:45:26.449898924Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:45:44.865622698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T21:46:05.142099187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36724"
ts=2026-05-12T21:46:14.870216314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:46:19.981246623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:46:24.648012765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:42002"
ts=2026-05-12T21:46:41.238026445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53170"
ts=2026-05-12T21:46:44.875982496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:47:11.23979438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34484"
ts=2026-05-12T21:47:14.877924357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:47:19.709605411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:47:41.241136573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33854"
ts=2026-05-12T21:47:44.88146798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:48:14.884560218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:48:19.663572663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:48:44.887660115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:48:59.743271228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:49:14.889043325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:49:24.671297139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:51208"
level=debug ts=2026-05-12T21:49:37.635594453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:49:37.637419928Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:49:44.890389208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:49:54.676151304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48394"
ts=2026-05-12T21:50:14.892264796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:50:18.976601861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:50:35.175872844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37546"
ts=2026-05-12T21:50:44.894268943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T21:50:54.68222427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:41470"
ts=2026-05-12T21:51:05.177654385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53640"
level=debug ts=2026-05-12T21:51:14.075446241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:51:14.896488487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T21:51:44.897692639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:51:54.688901548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40682"
ts=2026-05-12T21:52:05.18232408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54946"
ts=2026-05-12T21:52:11.257257126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:58032"
level=debug ts=2026-05-12T21:52:12.889379502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:52:14.898578314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T21:52:41.258838738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38774"
ts=2026-05-12T21:52:44.90080017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:52:59.812801181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:53:11.260488886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36074"
ts=2026-05-12T21:53:14.902620376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=error ts=2026-05-12T21:53:20.245023108Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:53:20.246917253Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:53:20.248665275Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T21:53:20.248702743Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:53:41.261927048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42996"
ts=2026-05-12T21:53:44.904461954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:54:01.10918875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:54:01.111005984Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:54:14.905721269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:54:44.906803346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:54:54.70805498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:59256"
level=debug ts=2026-05-12T21:54:54.898196638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:55:05.201674559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44014"
ts=2026-05-12T21:55:11.264851177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50576"
ts=2026-05-12T21:55:14.909110473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:55:24.711483859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:60478"
ts=2026-05-12T21:55:41.266199518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:57262"
ts=2026-05-12T21:55:44.910983832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T21:55:48.637508878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:55:54.714281705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34132"
ts=2026-05-12T21:56:14.912808694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:56:24.717248647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34024"
level=debug ts=2026-05-12T21:56:43.863443174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:56:44.915404873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T21:57:14.91719075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:57:37.393851182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:57:44.918985535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T21:57:54.727636178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:38016"
ts=2026-05-12T21:58:05.213859238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35178"
ts=2026-05-12T21:58:14.919982796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:58:16.411155207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T21:58:16.412850353Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T21:58:44.921785059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T21:58:56.004004235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:59:11.27884448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40348"
ts=2026-05-12T21:59:14.924389567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T21:59:41.280343705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:56390"
ts=2026-05-12T21:59:44.926818041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T21:59:52.51844317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T21:59:54.739769161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57214"
ts=2026-05-12T22:00:11.281993235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39354"
ts=2026-05-12T22:00:14.928543843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:00:24.742843297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:49030"
level=debug ts=2026-05-12T22:00:36.646851104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:00:44.93063497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:00:54.74583877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:44692"
ts=2026-05-12T22:01:14.932508955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:01:28.992372858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:01:35.230978701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35896"
ts=2026-05-12T22:01:41.289870011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45904"
ts=2026-05-12T22:01:44.934857093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:01:54.751582362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:38406"
ts=2026-05-12T22:02:05.232733243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44798"
ts=2026-05-12T22:02:14.936370038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:02:15.087431436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:02:15.089248902Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:02:35.234483945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54932"
ts=2026-05-12T22:02:44.93810433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:02:53.159202431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:03:14.938976903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=error ts=2026-05-12T22:03:20.241833338Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:03:20.243468342Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:03:20.245157502Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:03:20.245180595Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:03:35.237166574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39922"
ts=2026-05-12T22:03:41.295543814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52190"
ts=2026-05-12T22:03:44.940244058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:03:46.855218729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:04:14.941975501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:04:41.161744666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:04:44.942982598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:05:14.945152974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:05:32.288356854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:05:44.947496979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:05:54.777308187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54604"
ts=2026-05-12T22:06:05.250138814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39830"
ts=2026-05-12T22:06:14.949260296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:06:20.677801966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:06:20.679475319Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:06:35.25188422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34436"
ts=2026-05-12T22:06:44.951202995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:06:56.977100488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:07:14.953239328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:07:35.255176203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59696"
level=debug ts=2026-05-12T22:07:42.009318771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:07:44.956686081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:07:54.790404556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50446"
ts=2026-05-12T22:08:05.256535127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41740"
ts=2026-05-12T22:08:14.958431207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:08:19.280648528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:08:35.25796204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:57866"
ts=2026-05-12T22:08:41.317619988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45096"
ts=2026-05-12T22:08:44.959382158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:08:55.99734593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:09:14.960124448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:09:41.322127062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54292"
level=debug ts=2026-05-12T22:09:42.028222296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:09:42.02972142Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:09:44.961624652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:10:05.270115945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34220"
ts=2026-05-12T22:10:11.323927241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35744"
ts=2026-05-12T22:10:14.963410009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:10:24.805608065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50370"
ts=2026-05-12T22:10:41.325452967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50564"
level=debug ts=2026-05-12T22:10:41.40661958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:10:44.966501342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:11:05.274821665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:57408"
ts=2026-05-12T22:11:14.96883821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:11:24.812429375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:57922"
level=debug ts=2026-05-12T22:11:26.113139503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:11:44.970380017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:12:05.277554615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54708"
ts=2026-05-12T22:12:11.331601712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36614"
ts=2026-05-12T22:12:14.972727737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:12:19.062200565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:12:35.279192294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:48480"
ts=2026-05-12T22:12:44.974700863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:13:14.975837315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:13:18.394166477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:13:20.241236658Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:13:20.243247053Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:13:20.246099349Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:13:20.246123325Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:13:35.281580997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56730"
ts=2026-05-12T22:13:44.976999312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:14:04.375885397Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:14:04.377599905Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:14:05.283327528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33490"
ts=2026-05-12T22:14:14.97936567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:14:24.832174403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:59390"
ts=2026-05-12T22:14:35.28502122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52360"
ts=2026-05-12T22:14:44.98195558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:14:51.132810465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:15:14.983034314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:15:35.287802581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:34222"
level=debug ts=2026-05-12T22:15:38.995130194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:15:41.347853064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46256"
ts=2026-05-12T22:15:44.984511929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:15:54.842360566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45312"
ts=2026-05-12T22:16:14.986673974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:16:24.845846191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40110"
level=debug ts=2026-05-12T22:16:29.039584167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:16:41.351395647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:42208"
ts=2026-05-12T22:16:44.989389569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:16:54.848590569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:42702"
ts=2026-05-12T22:17:05.294381746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40240"
level=debug ts=2026-05-12T22:17:08.617008076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:17:14.991410301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:17:24.851292034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:41008"
ts=2026-05-12T22:17:44.993174968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:18:02.65175885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:18:02.65350555Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:18:14.994564782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:18:41.358103516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46808"
ts=2026-05-12T22:18:44.995395314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:18:49.750577684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:18:54.860878844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:38032"
ts=2026-05-12T22:19:14.996969547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:19:24.863785515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50650"
level=debug ts=2026-05-12T22:19:41.79451546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:19:44.999054709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:20:15.001492469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:20:21.276107277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:20:24.870680142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:46930"
ts=2026-05-12T22:20:45.002459174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:20:54.874208864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53848"
ts=2026-05-12T22:21:05.318393166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50358"
ts=2026-05-12T22:21:15.004143589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:21:21.974547614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:21:45.006196918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:21:59.278778385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:21:59.280301863Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:22:15.009262494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:22:35.328367493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33078"
ts=2026-05-12T22:22:45.012799938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:22:51.001975971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:23:15.016195176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=error ts=2026-05-12T22:23:20.24278052Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:23:20.244468912Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:23:20.246101067Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:23:20.246120991Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:23:35.905871454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:23:41.379256537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43452"
ts=2026-05-12T22:23:45.018927152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:23:54.892659939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:51460"
ts=2026-05-12T22:24:05.333582748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35048"
ts=2026-05-12T22:24:11.380185847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48202"
level=debug ts=2026-05-12T22:24:11.402200663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:24:15.021743484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:24:45.023421719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:24:58.98627166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:25:15.025172861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:25:45.026659457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:25:47.928905313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:25:47.930383408Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:26:05.342709445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41414"
ts=2026-05-12T22:26:15.02939695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:26:24.910602511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:55722"
ts=2026-05-12T22:26:35.344571823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52896"
level=debug ts=2026-05-12T22:26:35.471286493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:26:45.031722119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:27:11.465923777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:27:15.032719497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:27:24.916245886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:48794"
ts=2026-05-12T22:27:45.03358415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:27:54.919494038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:41446"
level=debug ts=2026-05-12T22:28:12.031915171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:28:15.034768346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:28:24.922513199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53850"
ts=2026-05-12T22:28:35.356178128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51542"
ts=2026-05-12T22:28:45.036326187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:29:00.647910157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:29:05.357268588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33730"
ts=2026-05-12T22:29:15.037744804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:29:35.358325486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52592"
ts=2026-05-12T22:29:45.038816267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:29:45.750762241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:29:45.752254503Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:30:15.040656847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:30:28.88460623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:30:41.412214645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:32836"
ts=2026-05-12T22:30:45.043779624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:30:54.938643703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:50924"
level=debug ts=2026-05-12T22:31:14.864474909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:31:15.046647708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:31:24.941537872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:33332"
ts=2026-05-12T22:31:45.047430248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:31:54.944617195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:34138"
level=debug ts=2026-05-12T22:32:10.801058757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:32:15.048657429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:32:45.0502378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:32:54.950779109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:44704"
level=debug ts=2026-05-12T22:32:56.704363072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:33:15.05173014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=error ts=2026-05-12T22:33:20.242963961Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:33:20.244995856Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:33:20.246895642Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:33:20.246917018Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:33:31.520556385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:33:31.522715088Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:33:45.053524126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:34:05.380322423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43444"
ts=2026-05-12T22:34:11.426216369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:36332"
ts=2026-05-12T22:34:15.055775155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:34:17.988515141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:34:41.42708541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49794"
ts=2026-05-12T22:34:45.058423965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:35:14.816656328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:35:15.061115403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:35:41.429187772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44216"
ts=2026-05-12T22:35:45.062282968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:35:54.968317254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:58670"
ts=2026-05-12T22:36:05.387828176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44308"
level=debug ts=2026-05-12T22:36:11.228585415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:36:15.063471451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:36:41.432042608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39900"
ts=2026-05-12T22:36:45.064966106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:37:01.178143765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:37:11.433649388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45986"
ts=2026-05-12T22:37:15.066001428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:37:35.394906051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54214"
ts=2026-05-12T22:37:41.435204694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51582"
level=debug ts=2026-05-12T22:37:41.940176637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:37:41.941778087Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:37:45.067295708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:37:54.979408737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:59774"
ts=2026-05-12T22:38:15.068934126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:38:24.982114685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:45286"
level=debug ts=2026-05-12T22:38:29.238644729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:38:45.071445633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:39:15.073739718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:39:24.98894655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:40428"
level=debug ts=2026-05-12T22:39:27.068436367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:39:45.075308873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:40:05.405542295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50380"
level=debug ts=2026-05-12T22:40:07.629926179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:40:15.076336142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:40:24.995355527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:56856"
ts=2026-05-12T22:40:35.406682555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60814"
ts=2026-05-12T22:40:41.449342356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38504"
ts=2026-05-12T22:40:45.077316915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:40:53.066651336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:41:05.408377633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51626"
ts=2026-05-12T22:41:15.078322104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:41:35.409471424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42926"
level=debug ts=2026-05-12T22:41:41.442867841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:41:41.444328954Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:41:41.451522018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54592"
ts=2026-05-12T22:41:45.079407906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:41:55.005265985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:59016"
ts=2026-05-12T22:42:11.452931605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39666"
ts=2026-05-12T22:42:15.080994768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:42:25.007802196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:35548"
level=debug ts=2026-05-12T22:42:33.684379305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:42:35.414044217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40000"
ts=2026-05-12T22:42:41.454234013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55624"
ts=2026-05-12T22:42:45.082452998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:42:55.01091508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54914"
ts=2026-05-12T22:43:11.455837208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46328"
ts=2026-05-12T22:43:15.083928344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=error ts=2026-05-12T22:43:20.241632292Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:43:20.243316322Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:43:20.244892873Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:43:20.244917081Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:43:36.1131677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:43:45.086174833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:44:12.234307482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:44:15.088365344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:44:25.019442714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:36130"
ts=2026-05-12T22:44:41.460281747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52002"
ts=2026-05-12T22:44:45.089137789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:45:02.716185168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:45:15.091286095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:45:43.204834929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:45:43.206567296Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:45:45.092898138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:46:11.468142168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49144"
ts=2026-05-12T22:46:15.094418111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:46:21.290720726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:46:41.46976356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46510"
ts=2026-05-12T22:46:45.097290686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:47:15.100188767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:47:15.265116414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:47:41.474036549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51222"
ts=2026-05-12T22:47:45.101958375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:47:55.042181409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:56996"
level=debug ts=2026-05-12T22:47:56.825379625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:48:15.102712186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:48:37.988850082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:48:45.105297587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:49:15.107360201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:49:25.051659376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:53582"
level=debug ts=2026-05-12T22:49:33.043519332Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:49:33.045196877Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:49:41.485253124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55070"
ts=2026-05-12T22:49:45.108999574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:50:05.4439214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37588"
ts=2026-05-12T22:50:15.110441201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:50:28.532591527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:50:35.444934471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43142"
ts=2026-05-12T22:50:45.111563761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:51:05.446150121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40096"
level=debug ts=2026-05-12T22:51:07.165530797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:51:15.113805317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:51:25.063874963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54002"
ts=2026-05-12T22:51:41.489993427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44030"
ts=2026-05-12T22:51:45.11651823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:52:04.340926645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:52:05.449208356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:47772"
ts=2026-05-12T22:52:15.118147051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
ts=2026-05-12T22:52:41.493711273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:46940"
ts=2026-05-12T22:52:45.119752155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:52:59.212376008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:53:11.494769474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:32924"
ts=2026-05-12T22:53:15.120658488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=error ts=2026-05-12T22:53:20.243805223Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:53:20.245650691Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:53:20.247401273Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-12T22:53:20.247430121Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-12T22:53:40.616376637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:53:40.618167467Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:53:41.495862306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33332"
ts=2026-05-12T22:53:45.123040497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:54:15.125098963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:54:24.614471202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:54:41.500898274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52194"
ts=2026-05-12T22:54:45.126995341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wlfrq-d754d947 10.129.0.71:7946"
level=debug ts=2026-05-12T22:55:05.409819786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:55:11.502694357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48404"
ts=2026-05-12T22:55:15.12939178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:55:45.131362587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:56:05.462461696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:56:05.466370397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56052"
ts=2026-05-12T22:56:11.505876134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34838"
ts=2026-05-12T22:56:15.133987166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
ts=2026-05-12T22:56:35.467951814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:48200"
ts=2026-05-12T22:56:45.135824581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
level=debug ts=2026-05-12T22:57:04.112315819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:57:05.468986197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35310"
ts=2026-05-12T22:57:15.137193944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:57:25.101729861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54598"
ts=2026-05-12T22:57:35.470250084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40956"
ts=2026-05-12T22:57:41.512520434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:44330"
ts=2026-05-12T22:57:45.139863669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:57:45.162460915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-12T22:57:45.16386623Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-12T22:58:05.471194535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35856"
ts=2026-05-12T22:58:11.513683279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45096"
ts=2026-05-12T22:58:15.142400761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:58:24.423867885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:58:25.107831723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:46144"
ts=2026-05-12T22:58:41.514873038Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:45108"
ts=2026-05-12T22:58:45.145814301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-wb7vb-616db018 10.128.0.60:7946"
level=debug ts=2026-05-12T22:59:07.786970154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-12T22:59:15.147869807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0eb69d50 10.129.0.72:7946"
ts=2026-05-12T22:59:25.114379664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.60:54862"