level=info ts=2026-02-16T11:31:00.816811376Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T11:31:00.816863657Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-02-16T11:31:00.818174023Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T11:31:00.818437111Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T11:31:00.818505291Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.61 inf=eth0
level=debug ts=2026-02-16T11:31:00.818959447Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T11:31:00.819032662Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.61 inf=eth0
level=info ts=2026-02-16T11:31:00.819046625Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-0285d981
ts=2026-02-16T11:31:00.819127689Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T11:31:00.819150685Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.61 advertisePort=7946
level=debug ts=2026-02-16T11:31:00.819171947Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.61 advertisePort=7946
level=info ts=2026-02-16T11:31:00.819322053Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-02-16T11:31:00.823243912Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:31:00.823257227Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T11:31:00.823265321Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=3.945244ms
level=error ts=2026-02-16T11:31:00.823271855Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-02-16T11:31:00.82328198Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-02-16T11:31:00.82530154Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:00.825315407Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T11:31:00.832522826Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T11:31:00.832533692Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-02-16T11:31:00.832555225Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-02-16T11:31:00.832553318Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T11:31:00.832607323Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T11:31:00.832631493Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T11:31:00.832682446Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-02-16T11:31:00.832700668Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-02-16T11:31:00.832777629Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-02-16T11:31:00.833066568Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-02-16T11:31:00.833125168Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-02-16T11:31:00.83311305Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-02-16T11:31:00.833225274Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T11:31:00.833398534Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-02-16T11:31:00.833406099Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-02-16T11:31:00.833416373Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-02-16T11:31:01.833638844Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-02-16T11:31:01.935871362Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-02-16T11:31:01.935923113Z caller=loki.go:599 msg="Loki started" startup_time=1.181429399s
ts=2026-02-16T11:31:02.280393619Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:02.280414037Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:06.188700069Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:06.188728504Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T11:31:06.936725509Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-02-16T11:31:06.936769452Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-02-16T11:31:11.046360486Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-02-16T11:31:11.046383869Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-02-16T11:31:19.579779949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:20.812740219Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.59:7946: dial tcp 10.130.0.59:7946: connect: connection refused"
ts=2026-02-16T11:31:20.814194616Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.71:7946: dial tcp 10.129.0.71:7946: connect: connection refused"
ts=2026-02-16T11:31:20.815051147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.69:7946"
level=info ts=2026-02-16T11:31:20.816257339Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=19.992970851s
level=debug ts=2026-02-16T11:31:37.896521683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:45.364609465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:31:56.698634542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:31:59.022718002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:60288"
ts=2026-02-16T11:32:15.36707669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:32:20.682223013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:32:30.132197883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39820"
ts=2026-02-16T11:32:45.369313688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:32:48.17085482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:32:48.172405683Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:33:15.371288483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:33:25.587639315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:33:45.372936574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:33:45.76125051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60928"
ts=2026-02-16T11:33:59.035335682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:55786"
ts=2026-02-16T11:34:00.136443956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43446"
ts=2026-02-16T11:34:15.375344039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:34:19.629479782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:34:29.03859711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39100"
ts=2026-02-16T11:34:45.377970729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:34:59.041504292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:42808"
level=debug ts=2026-02-16T11:35:13.559741955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:35:15.380748858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:35:45.382937558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:35:45.771597925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37500"
ts=2026-02-16T11:36:00.144347663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60346"
level=debug ts=2026-02-16T11:36:01.265901665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T11:36:15.38580452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:36:15.773593401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42212"
ts=2026-02-16T11:36:45.388673288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:36:45.776148902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49782"
ts=2026-02-16T11:36:59.053240809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:50506"
ts=2026-02-16T11:37:00.1484479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:37390"
level=debug ts=2026-02-16T11:37:04.90178662Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T11:37:04.90350578Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:37:15.391346007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:37:45.393813624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:37:45.780575109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53482"
level=debug ts=2026-02-16T11:37:57.681247347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:37:59.058678075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:57390"
ts=2026-02-16T11:38:00.151442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41768"
ts=2026-02-16T11:38:15.395823044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:38:35.141180641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:38:45.397020004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:39:15.397754388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:39:29.06789423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:53744"
level=debug ts=2026-02-16T11:39:35.440711716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:39:45.398774961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:40:15.400656288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:40:21.363626189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:40:45.403011149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=info ts=2026-02-16T11:41:06.936920711Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-02-16T11:41:06.942221443Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:41:06.944364945Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:41:06.946491232Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:41:06.946522375Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T11:41:06.946536218Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-02-16T11:41:14.102582972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:41:14.104206341Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:41:15.404937511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:41:45.406758224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:42:00.1690645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42944"
level=debug ts=2026-02-16T11:42:08.887345028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:42:15.409108946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:42:29.083341438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:44664"
ts=2026-02-16T11:42:45.411971301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:42:45.803003012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45394"
level=debug ts=2026-02-16T11:43:02.835049643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:43:15.413999525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:43:45.416066904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:43:55.980735641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:44:00.177668231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35606"
ts=2026-02-16T11:44:15.417622406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:44:29.095891998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:41710"
level=debug ts=2026-02-16T11:44:33.698941822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:44:45.419199797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:44:45.813689101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49120"
ts=2026-02-16T11:45:15.421998753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:45:33.322715637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:45:33.324356441Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:45:45.424210759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:46:15.427531647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:46:23.705111227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:46:45.430424611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:47:15.432934546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:47:23.517039162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:47:29.112780062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:50930"
ts=2026-02-16T11:47:45.43560599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:47:59.126333897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:51432"
ts=2026-02-16T11:48:15.437489236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:48:23.150866977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:48:29.129024827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:59794"
ts=2026-02-16T11:48:30.199251233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46822"
ts=2026-02-16T11:48:45.439229865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:49:00.200348918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54672"
ts=2026-02-16T11:49:15.441144609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:49:15.834544427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57338"
level=debug ts=2026-02-16T11:49:21.2991698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:49:45.443314218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:49:45.836921674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52248"
ts=2026-02-16T11:50:15.444666492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:50:15.83888243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:39856"
level=debug ts=2026-02-16T11:50:23.396784977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:50:23.398493078Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:50:29.140618564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40848"
ts=2026-02-16T11:50:45.446483117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:50:45.841376091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44032"
level=error ts=2026-02-16T11:51:06.952015522Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:51:06.954361647Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:51:06.956348006Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T11:51:06.956379887Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T11:51:10.378001793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:51:15.447725868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:51:29.14624835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39156"
ts=2026-02-16T11:51:30.211787636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34484"
ts=2026-02-16T11:51:45.449069377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:51:45.847492365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55114"
level=debug ts=2026-02-16T11:52:01.858383465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:52:15.450510678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:52:30.216373054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46708"
level=debug ts=2026-02-16T11:52:36.818674031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:52:45.452063731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:53:15.453590409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:53:15.85510911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51234"
level=debug ts=2026-02-16T11:53:30.599542951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:53:45.455069202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:54:05.911890195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:54:05.913571359Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:54:15.456551719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:54:15.859481898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57376"
ts=2026-02-16T11:54:29.164171821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:53758"
level=debug ts=2026-02-16T11:54:44.660996066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:54:45.45916131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T11:54:45.861828425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46046"
ts=2026-02-16T11:55:15.460740921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:55:40.59312973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:55:45.462830655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:55:45.865703984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45576"
ts=2026-02-16T11:56:15.465026112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T11:56:30.235360645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34414"
level=debug ts=2026-02-16T11:56:39.374265449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:56:45.467141373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:57:15.469025685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T11:57:15.495100031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:57:15.873883082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48284"
ts=2026-02-16T11:57:45.471139348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T11:57:50.028515809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T11:57:50.030437168Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T11:58:15.472807153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:58:15.879122595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54758"
level=debug ts=2026-02-16T11:58:42.300583991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:58:45.473858119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T11:58:45.881265718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51860"
ts=2026-02-16T11:58:59.189594228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:35642"
ts=2026-02-16T11:59:00.247217308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:32804"
ts=2026-02-16T11:59:15.475012762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T11:59:28.60586681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T11:59:45.476877988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:00:15.478961492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:00:20.423085008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:00:45.481117501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:01:00.256877776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:37442"
level=error ts=2026-02-16T12:01:06.951047481Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:01:06.953109498Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:01:06.954988067Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:01:06.95501441Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T12:01:09.08950583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:01:15.482676482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:01:29.204802666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:43800"
ts=2026-02-16T12:01:30.257765293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38690"
ts=2026-02-16T12:01:45.484683705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:01:45.898323534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56012"
level=debug ts=2026-02-16T12:01:55.55345018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:01:55.555153843Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:02:00.259445665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53438"
ts=2026-02-16T12:02:15.487013082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:02:29.21153307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:48262"
ts=2026-02-16T12:02:30.260953899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53460"
level=debug ts=2026-02-16T12:02:30.960018385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:02:45.489516676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:03:15.492460103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:03:16.392469038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:03:45.495650301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:03:45.90689935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56656"
level=debug ts=2026-02-16T12:03:52.909002367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:04:15.498333697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:04:31.741394167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:04:45.500197891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:04:59.228678017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:59760"
ts=2026-02-16T12:05:15.502197398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:05:15.913958681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:40042"
level=debug ts=2026-02-16T12:05:23.179336232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:05:23.180886125Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:05:45.504157067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:05:45.916090384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51802"
ts=2026-02-16T12:06:15.506096346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:06:23.808488045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:06:29.238214561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:47548"
ts=2026-02-16T12:06:45.507985425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:06:59.241643949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40592"
ts=2026-02-16T12:07:00.282755418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54242"
level=debug ts=2026-02-16T12:07:00.283567805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:07:15.509936932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:07:30.283737997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60352"
ts=2026-02-16T12:07:45.512480002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:08:02.068162589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:08:15.514068497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:08:15.936390744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43726"
level=debug ts=2026-02-16T12:08:38.763231649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:08:45.5163912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:08:45.938578515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58750"
ts=2026-02-16T12:08:59.252957902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:37378"
ts=2026-02-16T12:09:15.517969849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:09:29.256162899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:55376"
ts=2026-02-16T12:09:30.293358115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52360"
level=debug ts=2026-02-16T12:09:35.701903362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:09:35.703993005Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:09:45.52010944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:09:45.943806485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:53186"
ts=2026-02-16T12:10:15.522487895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:10:15.946333193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56706"
level=debug ts=2026-02-16T12:10:21.823924729Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:10:45.524522911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:11:00.300780442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35822"
level=error ts=2026-02-16T12:11:06.949225895Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:11:06.950759625Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:11:06.952141089Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:11:06.95217251Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-02-16T12:11:07.23343575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:11:15.527399302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:11:45.52902347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:11:45.954589606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:49020"
level=debug ts=2026-02-16T12:12:04.897813749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:12:15.531094418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:12:15.957016291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36794"
ts=2026-02-16T12:12:45.533280366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:13:06.772389445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:13:15.535876808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:13:15.962195135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36576"
ts=2026-02-16T12:13:30.309710414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:44500"
level=debug ts=2026-02-16T12:13:43.170289648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:13:43.171815883Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:13:45.538381295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:13:45.964537837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34830"
ts=2026-02-16T12:13:59.284512272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:55366"
ts=2026-02-16T12:14:00.310404465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59064"
ts=2026-02-16T12:14:15.541737567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:14:15.967007695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35630"
ts=2026-02-16T12:14:29.287310019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:50990"
level=debug ts=2026-02-16T12:14:31.141764221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:14:45.543749637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:14:59.290352962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:59222"
ts=2026-02-16T12:15:15.54473811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:15:15.971356418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55340"
level=debug ts=2026-02-16T12:15:28.424909304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:15:45.546326151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:16:15.547828248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:16:15.975825155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42422"
level=debug ts=2026-02-16T12:16:30.260868124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:16:45.549691646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:16:59.30133234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:32902"
ts=2026-02-16T12:17:15.551703817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:17:29.304692901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:46618"
ts=2026-02-16T12:17:30.324162739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57570"
level=debug ts=2026-02-16T12:17:31.564713477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:17:45.553006678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:17:45.983662017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37864"
level=debug ts=2026-02-16T12:18:10.041281487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:18:10.043026129Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:18:15.554734632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:18:15.985371903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41492"
ts=2026-02-16T12:18:29.310771572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:45586"
level=debug ts=2026-02-16T12:18:44.20453576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:18:45.55670628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:18:45.987284285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:33434"
ts=2026-02-16T12:18:59.313614681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:54082"
ts=2026-02-16T12:19:00.328943698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46408"
ts=2026-02-16T12:19:15.558704671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:19:42.203397086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:19:45.561064251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:20:15.564035772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:20:15.992993967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52610"
level=debug ts=2026-02-16T12:20:25.078895059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:20:30.334429551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60342"
ts=2026-02-16T12:20:45.566354148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:20:45.994463236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38824"
level=debug ts=2026-02-16T12:20:59.843638959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:21:06.949231075Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:21:06.951044606Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:21:06.952805237Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-02-16T12:21:06.952831948Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:21:15.568284281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
ts=2026-02-16T12:21:45.570217749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
level=debug ts=2026-02-16T12:21:59.836393491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-02-16T12:21:59.837816299Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-02-16T12:22:15.573201691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
ts=2026-02-16T12:22:16.001516141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51112"
ts=2026-02-16T12:22:29.334136228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:44190"
ts=2026-02-16T12:22:45.575875822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-czznf-71e050cb 10.129.0.69:7946"
level=debug ts=2026-02-16T12:22:55.688352352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-02-16T12:23:15.578138308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d875fc63 10.130.0.60:7946"
ts=2026-02-16T12:23:45.579806599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-zk8zd-d9910aa7 10.128.0.57:7946"
level=debug ts=2026-02-16T12:23:50.075832962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"