level=info ts=2026-05-13T12:36:48.383621157Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-13T12:36:48.383672428Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-13T12:36:48.38561213Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-13T12:36:48.386067147Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T12:36:48.386182327Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.54 inf=eth0
level=debug ts=2026-05-13T12:36:48.387358248Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T12:36:48.387445005Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.54 inf=eth0
level=info ts=2026-05-13T12:36:48.387505245Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-7c3801fa
ts=2026-05-13T12:36:48.387596607Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-13T12:36:48.387613334Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.54 advertisePort=7946
level=debug ts=2026-05-13T12:36:48.387638008Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.54 advertisePort=7946
level=info ts=2026-05-13T12:36:48.387808839Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-13T12:36:48.393628289Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T12:36:48.393647417Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:36:48.3936585Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.852136ms
level=error ts=2026-05-13T12:36:48.39366797Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-13T12:36:48.393679336Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-13T12:36:48.397149099Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:36:48.397167591Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T12:36:48.4046333Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-13T12:36:48.404691138Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-13T12:36:48.40471228Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-13T12:36:48.404736791Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-05-13T12:36:48.404747567Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-13T12:36:48.404776081Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-13T12:36:48.404881213Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-13T12:36:48.404892699Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-13T12:36:48.404899631Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-13T12:36:48.404903585Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-13T12:36:48.404975724Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-13T12:36:48.404993942Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-13T12:36:48.405020447Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-13T12:36:48.405146984Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-13T12:36:48.405155589Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-13T12:36:48.405169793Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-13T12:36:49.405589391Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-13T12:36:49.506863641Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-13T12:36:49.506922581Z caller=loki.go:615 msg="Loki started" startup_time=1.193035638s
ts=2026-05-13T12:36:49.731041279Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:36:49.731066905Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:36:52.080885411Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:36:52.080916527Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T12:36:54.507357744Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-13T12:36:54.507420476Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-13T12:36:57.940521779Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:36:57.940547017Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T12:37:06.56755212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:37:07.846055515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.54:7946"
ts=2026-05-13T12:37:07.846161826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.54:52600"
ts=2026-05-13T12:37:07.847304252Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.52:7946: dial tcp 10.129.0.52:7946: connect: connection refused"
ts=2026-05-13T12:37:07.847954659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.53:7946"
ts=2026-05-13T12:37:07.850581877Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.64:7946: dial tcp 10.130.0.64:7946: connect: connection refused"
ts=2026-05-13T12:37:07.852435306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.62:7946"
level=info ts=2026-05-13T12:37:07.854026972Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=19.460342987s
ts=2026-05-13T12:37:09.259884787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:53166"
ts=2026-05-13T12:37:09.58075159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:51242"
ts=2026-05-13T12:37:09.832999053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46562"
level=debug ts=2026-05-13T12:37:27.151853155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:37:27.808318232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:48774"
ts=2026-05-13T12:37:29.78917741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:37:34.947427907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54728"
level=debug ts=2026-05-13T12:37:43.816441832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:37:45.363894377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:39446"
ts=2026-05-13T12:37:59.792493236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T12:38:01.581955673Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:38:27.813553194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:60420"
level=debug ts=2026-05-13T12:38:28.393418117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T12:38:28.395447453Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:38:29.795296299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:38:59.797281859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T12:39:09.334857927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:39:29.799888918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:39:59.802135039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T12:40:09.522037135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:40:27.826979776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:50734"
ts=2026-05-13T12:40:29.805382353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:40:34.960838188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33794"
ts=2026-05-13T12:40:59.80808773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T12:41:14.454589799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:41:15.379963133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42032"
ts=2026-05-13T12:41:29.811537645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:41:59.814712387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:42:04.968338739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59274"
level=debug ts=2026-05-13T12:42:17.254472191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:42:29.818301475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:42:45.388462309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42144"
ts=2026-05-13T12:42:57.841682019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:52206"
ts=2026-05-13T12:42:59.820062102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:43:15.392013695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:33658"
level=debug ts=2026-05-13T12:43:25.711195976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T12:43:25.712895601Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:43:27.84275206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:41198"
ts=2026-05-13T12:43:29.822879946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:43:34.976463398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50754"
ts=2026-05-13T12:43:57.844379509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:35210"
ts=2026-05-13T12:43:59.825625339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:44:15.396759991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:34706"
level=debug ts=2026-05-13T12:44:28.144789394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:44:29.828244098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:44:34.981335455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:35554"
ts=2026-05-13T12:44:57.848704323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:51784"
ts=2026-05-13T12:44:59.830369243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T12:45:08.20121403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:45:15.402110328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:50132"
ts=2026-05-13T12:45:29.832457185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:45:34.98652852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56520"
level=debug ts=2026-05-13T12:45:55.322444956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:45:57.8533916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:37062"
ts=2026-05-13T12:45:59.835121913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:46:04.989154965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59478"
ts=2026-05-13T12:46:15.408654964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:50690"
ts=2026-05-13T12:46:27.854678446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:54822"
ts=2026-05-13T12:46:29.838605262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T12:46:30.068491487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:46:45.41215756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:48090"
level=info ts=2026-05-13T12:46:54.50796648Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-13T12:46:54.512924786Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:46:54.515233647Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:46:54.517077761Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:46:54.517108792Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T12:46:54.517121359Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-13T12:46:57.855911506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:35050"
ts=2026-05-13T12:46:59.841584163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:47:27.857601013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:43898"
level=debug ts=2026-05-13T12:47:28.271092781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T12:47:28.272639278Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:47:29.84415816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:47:34.996599679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50244"
ts=2026-05-13T12:47:45.418157178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:56952"
ts=2026-05-13T12:47:59.846801983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T12:48:09.283602421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:48:29.84963456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:48:45.424942366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:52374"
ts=2026-05-13T12:48:57.865358454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:43862"
ts=2026-05-13T12:48:59.852805665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:49:05.005661685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:60338"
level=debug ts=2026-05-13T12:49:07.956565927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:49:29.855974811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T12:49:42.321371945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:49:59.858210021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:50:05.0098185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36682"
ts=2026-05-13T12:50:15.432959495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:56426"
ts=2026-05-13T12:50:29.860299828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T12:50:40.99585037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:50:45.43673594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:55654"
ts=2026-05-13T12:50:59.862707771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:51:05.016132081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51476"
ts=2026-05-13T12:51:15.440303233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:59952"
ts=2026-05-13T12:51:29.865296781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T12:51:39.498093579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T12:51:39.499813351Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:51:45.443900717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53792"
ts=2026-05-13T12:51:57.882118998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:56178"
ts=2026-05-13T12:51:59.868444581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:52:05.023319927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54254"
level=debug ts=2026-05-13T12:52:22.523185971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:52:27.883305574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:41084"
ts=2026-05-13T12:52:29.871182659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:52:45.450103876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53738"
ts=2026-05-13T12:52:59.873581683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T12:53:16.451108316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:53:29.874926894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:53:35.033594747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48602"
ts=2026-05-13T12:53:59.877191895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:54:05.037010003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59790"
level=debug ts=2026-05-13T12:54:09.520749525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:54:15.458426686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:49126"
ts=2026-05-13T12:54:27.893898981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:54726"
ts=2026-05-13T12:54:29.880616989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:54:59.883877867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T12:55:01.556007177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:55:05.045448256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54728"
ts=2026-05-13T12:55:29.8866449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:55:45.469617207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:50190"
ts=2026-05-13T12:55:57.901936727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:43346"
level=debug ts=2026-05-13T12:55:58.564768774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T12:55:58.56703232Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:55:59.889367843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:56:27.903390913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59174"
ts=2026-05-13T12:56:29.893310111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T12:56:35.054482024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49982"
level=debug ts=2026-05-13T12:56:40.412190998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:56:45.478447856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53224"
level=error ts=2026-05-13T12:56:54.5219692Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:56:54.524642195Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:56:54.527215146Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T12:56:54.52724461Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:56:59.896831043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T12:57:17.809964831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:57:29.900503284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:57:35.060871284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49106"
level=debug ts=2026-05-13T12:57:57.963111576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:57:59.90259538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:58:05.064581756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40436"
ts=2026-05-13T12:58:15.486938671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47140"
ts=2026-05-13T12:58:29.904537596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T12:58:35.067688932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:52230"
ts=2026-05-13T12:58:57.921025864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:54714"
level=debug ts=2026-05-13T12:58:58.254388376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T12:58:59.907176748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T12:59:29.910763453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T12:59:37.95913576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T12:59:37.960744844Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:59:59.913288784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:00:13.117963522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:00:29.915870706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:00:56.675192691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:00:57.93246653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:58362"
ts=2026-05-13T13:00:59.918358871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:01:05.077364637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46616"
ts=2026-05-13T13:01:27.933554639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:38228"
ts=2026-05-13T13:01:29.919240802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:01:37.669774203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:01:59.921629115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:02:24.435660856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:02:29.92475058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:02:35.085485156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49490"
ts=2026-05-13T13:02:45.510480143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:38116"
ts=2026-05-13T13:02:59.927488639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:03:17.808666368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:03:17.810351101Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:03:29.930801599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:03:57.948219438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:44160"
ts=2026-05-13T13:03:59.933140534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:04:06.771207101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:04:29.935365246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:04:52.520243557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:04:59.93880577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:05:29.941494069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:05:53.599715817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:05:59.943757754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:06:15.529638133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:44684"
ts=2026-05-13T13:06:27.962099611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:47098"
ts=2026-05-13T13:06:29.946845168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:06:47.0808742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:06:54.520028561Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:06:54.521940099Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:06:54.523744099Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:06:54.523766347Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:06:59.949006417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:07:25.660912388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:07:25.662471188Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:07:27.966652909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:40544"
ts=2026-05-13T13:07:29.949911881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:07:57.967947519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59128"
ts=2026-05-13T13:07:59.952576636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:08:05.109042138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:43570"
level=debug ts=2026-05-13T13:08:18.733239953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:08:27.968993924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:51572"
ts=2026-05-13T13:08:29.955820303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:08:45.541730927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:39510"
level=debug ts=2026-05-13T13:08:55.630422542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:08:57.971012605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:41878"
ts=2026-05-13T13:08:59.958772732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:09:29.962445481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:09:45.227675714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:09:59.964475568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:10:05.120586179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:60704"
ts=2026-05-13T13:10:15.550972686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:55368"
level=debug ts=2026-05-13T13:10:25.361344511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:10:29.965607539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:10:59.967824482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:11:11.110068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:11:11.111873618Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:11:29.969817652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:11:35.126700422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:60434"
level=debug ts=2026-05-13T13:11:48.590951311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:11:57.98899443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:55536"
ts=2026-05-13T13:11:59.972092043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:12:29.975863524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:12:41.965184648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:12:45.56428824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:59276"
ts=2026-05-13T13:12:59.978999441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:13:27.996720541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:55788"
ts=2026-05-13T13:13:29.98184101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:13:35.141737598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55288"
level=debug ts=2026-05-13T13:13:37.550968597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:13:59.983943324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:14:20.312768996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:14:29.985939474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:14:59.988897574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:15:00.899827605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:15:00.901391134Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:15:29.99184824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:15:35.149676297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59498"
ts=2026-05-13T13:15:45.57916859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47832"
level=debug ts=2026-05-13T13:15:47.603968622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:15:59.993833074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:16:29.995530206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:16:36.673316795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:16:54.521878994Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:16:54.523835098Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:16:54.525967717Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:16:54.525990836Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:16:59.997891874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:17:18.651834996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:17:30.000052389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:18:00.002526974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:18:05.431334853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:18:15.591840232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:40096"
ts=2026-05-13T13:18:30.004458475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:18:35.16612741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:57576"
level=debug ts=2026-05-13T13:18:54.688523783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:18:54.690167918Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:19:00.00548631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:19:30.007942044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:19:35.171118574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48276"
level=debug ts=2026-05-13T13:19:40.115030359Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:19:45.598237041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:37724"
ts=2026-05-13T13:20:00.010791044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:20:19.461017698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:20:30.012816724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:20:45.604648049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:37972"
ts=2026-05-13T13:21:00.015909964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:21:04.814755993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:21:30.019152219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:21:45.611417896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:40498"
ts=2026-05-13T13:22:00.020903706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:22:05.18582378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56922"
level=debug ts=2026-05-13T13:22:05.92494316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:22:28.053593773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59960"
ts=2026-05-13T13:22:30.023635112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:22:35.188729983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:47696"
ts=2026-05-13T13:23:00.026217312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:23:02.050420557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:23:02.052392182Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:23:30.028884391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:23:55.601044281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:24:00.031460187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:24:30.032852622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:24:45.625018889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:52462"
level=debug ts=2026-05-13T13:24:46.908537564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:25:00.035280684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:25:05.200355379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54394"
ts=2026-05-13T13:25:15.628110065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:59936"
ts=2026-05-13T13:25:28.07057579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:48990"
ts=2026-05-13T13:25:30.038645528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:25:45.211550324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:25:58.071852441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:47098"
ts=2026-05-13T13:26:00.041201775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:26:30.04228988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:26:38.527149117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:26:45.63471861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:41742"
level=error ts=2026-05-13T13:26:54.520937842Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:26:54.522608573Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:26:54.52498673Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:26:54.525011158Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:27:00.043218655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:27:18.517534343Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:27:18.519107227Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:27:28.079416891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:47812"
ts=2026-05-13T13:27:30.044622199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:28:00.047348957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:28:05.214492162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50276"
level=debug ts=2026-05-13T13:28:08.611368874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:28:30.050183151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:28:45.645457986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:55868"
ts=2026-05-13T13:28:58.08750191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:47986"
ts=2026-05-13T13:29:00.053329265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:29:05.805966833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:29:15.648592544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:55190"
ts=2026-05-13T13:29:28.089030211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:42232"
ts=2026-05-13T13:29:30.057412061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:29:45.159937059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:30:00.05960749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:30:19.508440312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:30:28.093664435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:41330"
ts=2026-05-13T13:30:30.062136156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:30:35.227411031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56534"
ts=2026-05-13T13:30:45.654789763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53308"
ts=2026-05-13T13:31:00.064230721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:31:14.61162817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:31:14.613213352Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:31:28.09843797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:53104"
ts=2026-05-13T13:31:30.067128953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:31:35.231315523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33896"
ts=2026-05-13T13:31:58.099748012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:38452"
ts=2026-05-13T13:32:00.06987001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:32:04.923318383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:32:30.073055028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:32:50.38283858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:33:00.076209072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:33:05.239176506Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37694"
ts=2026-05-13T13:33:30.079469573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:33:45.670381412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:52458"
level=debug ts=2026-05-13T13:33:47.662924316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:33:58.111207443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:44988"
ts=2026-05-13T13:34:00.082573694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:34:05.24377234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39596"
ts=2026-05-13T13:34:28.112348595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:58252"
ts=2026-05-13T13:34:30.085270213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:34:43.423307337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:35:00.087484049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:35:30.090210243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:35:33.84938719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:35:33.851013523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:36:00.092005306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:36:12.568515547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:36:15.683354868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:39912"
ts=2026-05-13T13:36:28.122912611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:51138"
ts=2026-05-13T13:36:30.092935326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:36:35.258284249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:58108"
ts=2026-05-13T13:36:45.686462277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:54344"
level=error ts=2026-05-13T13:36:54.521975844Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:36:54.523985599Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:36:54.525996275Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:36:54.526020014Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:37:00.093915142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:37:07.054359842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:37:30.09646465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:37:35.26506572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39708"
ts=2026-05-13T13:37:45.692929591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42896"
ts=2026-05-13T13:38:00.098460188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:38:04.889441254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:38:30.100366225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:38:45.697735609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:54520"
ts=2026-05-13T13:38:58.138691985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:55784"
ts=2026-05-13T13:39:00.105012951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:39:04.188806971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:39:15.701348059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:35248"
ts=2026-05-13T13:39:28.140140107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:35814"
ts=2026-05-13T13:39:30.108064187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:39:35.275588592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:51174"
ts=2026-05-13T13:39:45.705170758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:37316"
ts=2026-05-13T13:40:00.110961139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:40:00.574073568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:40:00.575890374Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:40:30.114484761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:40:43.658422335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:41:00.117880645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:41:05.280895869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34810"
ts=2026-05-13T13:41:15.711736255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:41928"
ts=2026-05-13T13:41:28.151875075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59258"
ts=2026-05-13T13:41:30.121020858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:41:44.940944088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:41:58.15351616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:51730"
ts=2026-05-13T13:42:00.124693457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:42:05.287124362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55410"
ts=2026-05-13T13:42:30.127374818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:42:36.423345485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:42:45.719878421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:41114"
ts=2026-05-13T13:43:00.129685883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:43:22.022984021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:43:28.160628215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:40022"
ts=2026-05-13T13:43:30.132029936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:44:00.134736503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:44:05.296612098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:43888"
level=debug ts=2026-05-13T13:44:10.139303954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:44:10.140910029Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:44:15.727539633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:34360"
ts=2026-05-13T13:44:30.136925743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:44:35.299771804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45860"
ts=2026-05-13T13:44:58.168278752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:48202"
ts=2026-05-13T13:45:00.138645022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:45:00.738826179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:45:05.302708067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41514"
ts=2026-05-13T13:45:30.139981472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:45:35.30597961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:36482"
level=debug ts=2026-05-13T13:45:40.237504011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:45:58.173299313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:33188"
ts=2026-05-13T13:46:00.141425146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:46:30.143469839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:46:35.311374186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:42658"
level=debug ts=2026-05-13T13:46:42.717721364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:46:45.741017003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:35578"
level=error ts=2026-05-13T13:46:54.522510326Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:46:54.524430288Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:46:54.5260642Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:46:54.526083594Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:47:00.14693504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:47:05.315028023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59248"
ts=2026-05-13T13:47:30.149853255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:47:39.485554661Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:48:00.152763132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:48:21.053740965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:48:21.055561064Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:48:28.189930991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:43306"
ts=2026-05-13T13:48:30.156152751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:49:00.158383057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:49:13.771548137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:49:30.160056062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:49:35.33062061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45538"
level=debug ts=2026-05-13T13:49:55.366721363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:49:58.196721943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:34078"
ts=2026-05-13T13:50:00.161623223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:50:30.163729652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T13:50:43.387627868Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:51:00.166546107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:51:05.346732472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55778"
ts=2026-05-13T13:51:15.75892875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:41320"
level=debug ts=2026-05-13T13:51:28.933101999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:51:30.168349431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:51:35.350166752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39002"
ts=2026-05-13T13:51:45.762566418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:40990"
ts=2026-05-13T13:51:58.207529057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:36080"
ts=2026-05-13T13:52:00.170794355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:52:23.825443971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:52:23.827035994Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:52:30.173758812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:52:35.357125759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:41794"
ts=2026-05-13T13:53:00.177352902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:53:05.360071777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46774"
level=debug ts=2026-05-13T13:53:15.086555331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:53:15.768817667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:47340"
ts=2026-05-13T13:53:28.215544646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:38984"
ts=2026-05-13T13:53:30.179575084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:53:35.363103236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:44330"
ts=2026-05-13T13:54:00.181936704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:54:05.366477677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:56984"
level=debug ts=2026-05-13T13:54:10.600954625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:54:15.773512468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:46694"
ts=2026-05-13T13:54:28.219699548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:36276"
ts=2026-05-13T13:54:30.185699927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:54:35.369403071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:54314"
level=debug ts=2026-05-13T13:54:59.18434038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:55:00.189236875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T13:55:15.779410222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:55688"
ts=2026-05-13T13:55:30.192175351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:55:45.783041957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:58686"
level=debug ts=2026-05-13T13:55:50.81322408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:55:58.227751398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:52390"
ts=2026-05-13T13:56:00.194343098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:56:28.228802732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:50530"
ts=2026-05-13T13:56:30.196973947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:56:36.121761448Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:56:36.123648603Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:56:45.789490845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:40822"
level=error ts=2026-05-13T13:56:54.52183047Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:56:54.523846038Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:56:54.525752247Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T13:56:54.525770939Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T13:57:00.199101676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T13:57:15.242669735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:57:28.233337196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:47616"
ts=2026-05-13T13:57:30.201594587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:57:45.795576879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:45162"
ts=2026-05-13T13:58:00.204827387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T13:58:11.906231003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:58:15.798836008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:44762"
ts=2026-05-13T13:58:28.238068164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:56548"
ts=2026-05-13T13:58:30.207212698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T13:58:35.387603021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:49606"
ts=2026-05-13T13:58:45.801756666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:41046"
level=debug ts=2026-05-13T13:58:46.59208441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:58:58.239596784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:34958"
ts=2026-05-13T13:59:00.209233106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:59:28.240952582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:40834"
ts=2026-05-13T13:59:30.212655156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T13:59:35.393783753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40966"
level=debug ts=2026-05-13T13:59:48.01728409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:59:58.242849045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:58948"
ts=2026-05-13T14:00:00.216024646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:00:05.396667796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50872"
ts=2026-05-13T14:00:15.808716021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:33322"
level=debug ts=2026-05-13T14:00:22.529549749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:00:22.531313399Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:00:28.243935743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:39928"
ts=2026-05-13T14:00:30.219264866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T14:01:00.222353038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T14:01:05.401500507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:38004"
level=debug ts=2026-05-13T14:01:23.447188348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:01:30.225292605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T14:02:00.229245713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:02:05.407543955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:39524"
level=debug ts=2026-05-13T14:02:10.704881826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:02:28.255647027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:33398"
ts=2026-05-13T14:02:30.231242027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:02:58.257294733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:47190"
ts=2026-05-13T14:03:00.232248612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:03:05.414236576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:48464"
level=debug ts=2026-05-13T14:03:07.900573599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:03:30.234779284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T14:03:57.668846242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:04:00.237323608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:04:30.238895629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T14:04:39.000288073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:04:39.002131713Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:04:45.831073057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:35092"
ts=2026-05-13T14:05:00.241681305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T14:05:28.270916472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:55906"
level=debug ts=2026-05-13T14:05:29.943040398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:05:30.244051762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:06:00.246753791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:06:05.428527876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:45140"
level=debug ts=2026-05-13T14:06:07.823112213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:06:30.250156736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:06:45.840453939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:48320"
level=debug ts=2026-05-13T14:06:50.442035817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:06:54.519270989Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:06:54.520830269Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:06:54.522340942Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:06:54.522361334Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:07:00.252559279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:07:05.433119592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:53870"
ts=2026-05-13T14:07:28.281954843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:32924"
ts=2026-05-13T14:07:30.254811399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T14:07:45.012411498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:07:58.2831857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:36218"
ts=2026-05-13T14:08:00.257301845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:08:28.284237547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59512"
ts=2026-05-13T14:08:30.259835665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T14:08:31.630141418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:08:31.631959215Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:08:35.441149528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59772"
ts=2026-05-13T14:09:00.26318232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T14:09:14.458344141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:09:15.851835014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:42544"
ts=2026-05-13T14:09:28.289124633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:39930"
ts=2026-05-13T14:09:30.26607509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:09:45.854995551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:46546"
ts=2026-05-13T14:09:58.290695591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:46752"
ts=2026-05-13T14:10:00.267734041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
level=debug ts=2026-05-13T14:10:08.792101894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:10:30.269382515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:10:35.451803717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:33966"
ts=2026-05-13T14:10:58.295086885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59896"
ts=2026-05-13T14:11:00.271395521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:11:05.454787807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:55596"
level=debug ts=2026-05-13T14:11:06.015925646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:11:28.296605119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:59778"
ts=2026-05-13T14:11:30.274078912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T14:11:35.457921682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37318"
ts=2026-05-13T14:11:45.864788138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:54454"
ts=2026-05-13T14:12:00.27787288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
level=debug ts=2026-05-13T14:12:05.928284232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:12:15.868118704Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:51016"
ts=2026-05-13T14:12:30.281026574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T14:12:58.306409041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:40262"
level=debug ts=2026-05-13T14:12:59.909579122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:12:59.911244467Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:13:00.283706444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:13:28.310314198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:55180"
ts=2026-05-13T14:13:30.285773061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:13:35.46676336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:34886"
level=debug ts=2026-05-13T14:13:38.096792627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:14:00.288828443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
level=debug ts=2026-05-13T14:14:13.050570612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:14:15.876985321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:43574"
ts=2026-05-13T14:14:28.315418554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.53:48806"
ts=2026-05-13T14:14:30.291464878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-q75rf-f3cccdd9 10.129.0.53:7946"
ts=2026-05-13T14:14:35.4716897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:46262"
ts=2026-05-13T14:14:45.880022881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:53766"
ts=2026-05-13T14:15:00.293657367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T14:15:05.475019993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:40640"
level=debug ts=2026-05-13T14:15:06.257797521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:15:30.296454789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:15:35.478523166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59396"
level=debug ts=2026-05-13T14:15:56.032306713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:16:00.299387216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:16:30.3023265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-afde4f5b 10.130.0.65:7946"
ts=2026-05-13T14:16:35.483558309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:50994"
level=debug ts=2026-05-13T14:16:48.044205494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:16:48.045864511Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:16:54.520162574Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:16:54.522014917Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:16:54.523867076Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-13T14:16:54.5238931Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T14:17:00.305237198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-rz79s-790e2aed 10.130.0.62:7946"
ts=2026-05-13T14:17:05.487277121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:37356"