level=info ts=2026-05-06T04:38:03.699094401Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-06T04:38:03.699132729Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-06T04:38:03.700694483Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-06T04:38:03.701009131Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T04:38:03.701082962Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.62 inf=eth0
level=debug ts=2026-05-06T04:38:03.701804294Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T04:38:03.701872948Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.62 inf=eth0
level=info ts=2026-05-06T04:38:03.701967507Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-7215ab5b
ts=2026-05-06T04:38:03.702036212Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-06T04:38:03.702053226Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.62 advertisePort=7946
level=debug ts=2026-05-06T04:38:03.702068424Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.62 advertisePort=7946
level=info ts=2026-05-06T04:38:03.702182225Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-06T04:38:03.708369969Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T04:38:03.708387113Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-06T04:38:03.708396113Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.215643ms
level=error ts=2026-05-06T04:38:03.708406165Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-06T04:38:03.708419753Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-06T04:38:03.711171326Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T04:38:03.711190245Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T04:38:03.717411356Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-06T04:38:03.717424221Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-06T04:38:03.717448296Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-06T04:38:03.717438795Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-06T04:38:03.717462936Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-06T04:38:03.717547672Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-06T04:38:03.717549776Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-06T04:38:03.717561141Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-06T04:38:03.717571334Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-06T04:38:03.717721485Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-06T04:38:03.717753986Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-06T04:38:03.717809038Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-06T04:38:03.717824619Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-06T04:38:03.717957032Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-06T04:38:03.717966878Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-06T04:38:03.71798126Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-06T04:38:04.718437633Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-06T04:38:04.915978797Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-06T04:38:04.916027498Z caller=loki.go:615 msg="Loki started" startup_time=1.280703468s
ts=2026-05-06T04:38:05.513602195Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T04:38:05.513640674Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:38:09.281044438Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T04:38:09.281067276Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T04:38:09.916838815Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-06T04:38:09.916898515Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-06T04:38:15.921890925Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T04:38:15.921914816Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:38:23.750704314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58066"
level=debug ts=2026-05-06T04:38:24.624625172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:38:24.930692509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35872"
ts=2026-05-06T04:38:30.595134755Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.43:7946: dial tcp 10.128.0.43:7946: connect: connection refused"
ts=2026-05-06T04:38:30.595755865Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.42:7946: dial tcp 10.128.0.42:7946: connect: connection refused"
ts=2026-05-06T04:38:30.596362664Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.44:7946: dial tcp 10.128.0.44:7946: connect: connection refused"
ts=2026-05-06T04:38:30.596452527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.62:7946"
ts=2026-05-06T04:38:30.596467302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.62:56048"
ts=2026-05-06T04:38:30.5978634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
ts=2026-05-06T04:38:30.600001164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
level=info ts=2026-05-06T04:38:30.601144657Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.892718752s
ts=2026-05-06T04:38:32.343910443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:34168"
level=debug ts=2026-05-06T04:38:44.606129332Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:38:50.951443737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60674"
ts=2026-05-06T04:38:55.435420921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:39:04.810377586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:55854"
level=debug ts=2026-05-06T04:39:06.378730786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:39:20.954892313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33752"
ts=2026-05-06T04:39:25.438012849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T04:39:29.059866134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:39:34.81326693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46868"
ts=2026-05-06T04:39:40.644006755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35000"
ts=2026-05-06T04:39:55.441431681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T04:40:03.016072695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T04:40:03.01770771Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T04:40:10.645262956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47844"
ts=2026-05-06T04:40:25.443439288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T04:40:37.739554113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:40:55.445480349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T04:41:04.82035583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:50286"
ts=2026-05-06T04:41:10.649939958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47972"
ts=2026-05-06T04:41:20.966534849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42176"
ts=2026-05-06T04:41:25.447187233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T04:41:26.355653672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:41:40.650737591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36882"
ts=2026-05-06T04:41:55.448325351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:42:20.97078079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53326"
ts=2026-05-06T04:42:25.450312049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T04:42:29.841310961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:42:34.827255957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:52600"
ts=2026-05-06T04:42:40.654863015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59126"
ts=2026-05-06T04:42:55.45316339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T04:43:04.830270437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:38168"
ts=2026-05-06T04:43:25.456309385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T04:43:40.659690947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42916"
level=debug ts=2026-05-06T04:43:43.642275096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T04:43:55.459719401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T04:44:25.462926063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T04:44:49.52072885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T04:44:49.52220549Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T04:44:55.465928918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T04:45:10.666892077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55318"
ts=2026-05-06T04:45:20.983089962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54280"
ts=2026-05-06T04:45:25.468772257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T04:45:44.866598933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:45:55.471217295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:46:20.989369797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55100"
ts=2026-05-06T04:46:25.474007017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T04:46:30.4681235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:46:34.850491196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33434"
ts=2026-05-06T04:46:40.673297464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58924"
ts=2026-05-06T04:46:55.477539635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T04:47:20.994504586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38064"
ts=2026-05-06T04:47:25.480391944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T04:47:25.736489773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:47:55.482844958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T04:48:00.96157132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:48:04.856352295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:41206"
level=info ts=2026-05-06T04:48:09.917838671Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-06T04:48:09.922711061Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T04:48:09.924585826Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T04:48:09.926412364Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T04:48:09.926433457Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T04:48:09.926445678Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-06T04:48:25.486006532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T04:48:43.940916395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T04:48:43.942394655Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T04:48:55.488215128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:49:21.002771157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60768"
ts=2026-05-06T04:49:25.490912495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T04:49:34.865905183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46484"
level=debug ts=2026-05-06T04:49:35.982677916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:49:55.49396917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T04:50:04.868895548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:40404"
level=debug ts=2026-05-06T04:50:21.282521606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:50:25.495875752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:50:55.497687153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T04:51:10.701730732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:53374"
level=debug ts=2026-05-06T04:51:15.197878821Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:51:21.012316831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44022"
ts=2026-05-06T04:51:25.500309601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:51:34.874979485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:56118"
level=debug ts=2026-05-06T04:51:55.151624667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:51:55.502547252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T04:52:25.505726519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T04:52:34.880541793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:53418"
level=debug ts=2026-05-06T04:52:48.361759754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T04:52:48.363716539Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T04:52:51.020490271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37660"
ts=2026-05-06T04:52:55.50919993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T04:53:04.883540678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:47422"
ts=2026-05-06T04:53:21.023364922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56140"
ts=2026-05-06T04:53:25.510908805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T04:53:36.350979327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:53:40.720321799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55622"
ts=2026-05-06T04:53:55.512713569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T04:54:11.744251864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:54:21.029325944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60064"
ts=2026-05-06T04:54:25.517774522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:54:40.724567152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34348"
level=debug ts=2026-05-06T04:54:48.646189291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:54:55.521184345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:55:04.8930799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:47740"
ts=2026-05-06T04:55:21.034270834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50774"
ts=2026-05-06T04:55:25.522443005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T04:55:34.896419891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43044"
level=debug ts=2026-05-06T04:55:46.078838206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:55:51.03781179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43498"
ts=2026-05-06T04:55:55.524824867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T04:56:21.041366004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46698"
ts=2026-05-06T04:56:25.528201866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T04:56:40.73622133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42928"
level=debug ts=2026-05-06T04:56:42.70273874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T04:56:42.704404725Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T04:56:51.044349525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58062"
ts=2026-05-06T04:56:55.531691924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T04:57:25.534621221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T04:57:28.803433168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:57:51.048173995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48864"
ts=2026-05-06T04:57:55.537851402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T04:58:06.141921972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T04:58:09.933584171Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T04:58:09.93748215Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T04:58:09.940004938Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T04:58:09.940032385Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T04:58:10.743754561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34664"
ts=2026-05-06T04:58:25.540963432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T04:58:55.544333215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T04:59:01.586171931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:59:04.911258505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:40554"
ts=2026-05-06T04:59:25.547490015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T04:59:37.944363235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T04:59:55.549841838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:00:04.916820605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:36650"
ts=2026-05-06T05:00:10.755741801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52352"
ts=2026-05-06T05:00:21.058826475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57388"
ts=2026-05-06T05:00:25.552234243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T05:00:38.183942707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:00:38.185857784Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:00:55.554621829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:01:21.065737162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41760"
ts=2026-05-06T05:01:25.557150244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T05:01:26.277490715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:01:55.560756425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:02:04.928962572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33816"
level=debug ts=2026-05-06T05:02:14.829672014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:02:25.564007413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:02:40.769760509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60986"
ts=2026-05-06T05:02:55.566875502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:03:09.004483538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:03:10.771291579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47114"
ts=2026-05-06T05:03:25.570010067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:03:55.571831066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:04:10.351589046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:04:25.573597364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:04:52.133855711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:04:52.136367028Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:04:55.575461812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:05:10.781986401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45052"
ts=2026-05-06T05:05:25.578038422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:05:51.084982211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59990"
level=debug ts=2026-05-06T05:05:52.828521667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:05:55.581187211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:06:25.58449715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:06:27.65001863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:06:51.08969388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53240"
ts=2026-05-06T05:06:55.587816704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:07:04.95364124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:52686"
level=debug ts=2026-05-06T05:07:06.482156195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:07:25.589618454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:07:51.093565204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58022"
ts=2026-05-06T05:07:55.590652115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:08:03.688802497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:08:04.960209966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:49676"
level=error ts=2026-05-06T05:08:09.929734655Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:08:09.93186215Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:08:09.933721565Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:08:09.933745633Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:08:21.096461179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41970"
ts=2026-05-06T05:08:25.591783351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:08:38.064241603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:08:38.065981466Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:08:55.59375185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:09:10.806350378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56206"
level=debug ts=2026-05-06T05:09:16.182750288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:09:25.596517402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:09:34.96780547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:35060"
ts=2026-05-06T05:09:51.104279586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59218"
ts=2026-05-06T05:09:55.599429327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T05:10:00.132837074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:10:25.602928398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:10:34.972830759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:41676"
level=debug ts=2026-05-06T05:10:41.374406062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:10:51.108821839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50470"
ts=2026-05-06T05:10:55.605970649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:11:23.609353713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:11:25.608729612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:11:40.82773108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37968"
ts=2026-05-06T05:11:51.112652893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60894"
ts=2026-05-06T05:11:55.611323031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:12:09.904518733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:12:09.906394011Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:12:10.828896686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44528"
ts=2026-05-06T05:12:25.614787453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:12:34.984874029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57658"
ts=2026-05-06T05:12:51.117118425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50642"
ts=2026-05-06T05:12:55.616869775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:13:05.724007282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:13:21.119747557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46012"
ts=2026-05-06T05:13:25.619422515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:13:51.123153147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59880"
ts=2026-05-06T05:13:55.622319618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:14:03.450827295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:14:10.839452212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50548"
ts=2026-05-06T05:14:25.62491186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:14:52.941244708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:14:55.627020549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:15:04.995734075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:44130"
ts=2026-05-06T05:15:10.843852404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41042"
ts=2026-05-06T05:15:21.130745019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35190"
ts=2026-05-06T05:15:25.628801505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:15:33.37562675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:15:40.845271415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36188"
ts=2026-05-06T05:15:51.133072668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37124"
ts=2026-05-06T05:15:55.629990564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:16:09.344645016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:16:09.346108479Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:16:25.631803405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:16:40.849422108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52072"
ts=2026-05-06T05:16:55.634876535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:17:10.851013197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52706"
level=debug ts=2026-05-06T05:17:10.904875361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:17:25.637878628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:17:40.852374442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48766"
level=debug ts=2026-05-06T05:17:50.729020745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:17:55.641280653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=error ts=2026-05-06T05:18:09.933762852Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:18:09.936317354Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:18:09.938664312Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:18:09.938697564Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:18:21.144752169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36710"
ts=2026-05-06T05:18:25.64413005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:18:52.437302517Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:18:55.646764734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:19:05.013811409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:39272"
ts=2026-05-06T05:19:10.859194197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35750"
ts=2026-05-06T05:19:21.150991181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41332"
ts=2026-05-06T05:19:25.648697357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:19:40.860268288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45024"
level=debug ts=2026-05-06T05:19:47.132343761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:19:55.651304478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:20:21.155450824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37280"
ts=2026-05-06T05:20:25.65385157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:20:35.019632927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43132"
level=debug ts=2026-05-06T05:20:37.941713676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:20:37.944035216Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:20:55.656921352Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:21:05.030750646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:37502"
level=debug ts=2026-05-06T05:21:23.158175214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:21:25.660021516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:21:55.663278846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T05:22:02.40634796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:22:25.666802483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:22:40.877087859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59536"
ts=2026-05-06T05:22:51.170231145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41172"
ts=2026-05-06T05:22:55.669154592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:22:58.306664377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:23:10.87857396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:46852"
ts=2026-05-06T05:23:25.670790404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:23:38.32268042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:23:51.175544221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42002"
ts=2026-05-06T05:23:55.671745531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:24:15.908727167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:24:15.910377575Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:24:21.178853512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44250"
ts=2026-05-06T05:24:25.673297199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:24:40.885777721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57848"
ts=2026-05-06T05:24:55.675056217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:25:18.02433363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:25:25.677501188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:25:51.184913159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44788"
ts=2026-05-06T05:25:55.680734674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:26:09.786710162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:26:25.68349358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:26:35.054542519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:55986"
level=debug ts=2026-05-06T05:26:50.984100563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:26:51.191273548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50690"
ts=2026-05-06T05:26:55.686357127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:27:10.89883697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48308"
ts=2026-05-06T05:27:25.689691361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:27:36.338972566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:27:40.90030464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38304"
ts=2026-05-06T05:27:51.196379612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53042"
ts=2026-05-06T05:27:55.6915889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=error ts=2026-05-06T05:28:09.932073947Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:28:09.934289824Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:28:09.93645039Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:28:09.936479696Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:28:10.901849717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56184"
level=debug ts=2026-05-06T05:28:15.969358146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:28:15.971115077Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:28:21.198956283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52426"
ts=2026-05-06T05:28:25.694051291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:28:35.065649277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:48894"
ts=2026-05-06T05:28:55.697442478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T05:28:55.817703421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:29:05.068982509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45966"
ts=2026-05-06T05:29:25.699565822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:29:55.701265491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:29:56.188788927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:30:25.702942768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:30:35.074384985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:59732"
ts=2026-05-06T05:30:40.913162078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49408"
ts=2026-05-06T05:30:55.705213694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:30:57.242746732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:31:25.706401727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:31:43.048195282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:31:55.709110882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:32:10.920943801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:53054"
ts=2026-05-06T05:32:21.215758066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60470"
ts=2026-05-06T05:32:25.711005949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:32:37.888931354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:32:37.890443519Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:32:55.713071896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:33:05.086575328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:42920"
ts=2026-05-06T05:33:25.716032234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:33:38.337891424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:33:51.22310755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48706"
ts=2026-05-06T05:33:55.718529714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:34:21.190557493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:34:25.720309113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:34:35.096913127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46072"
ts=2026-05-06T05:34:51.22905083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50662"
ts=2026-05-06T05:34:55.723324596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:35:05.099706203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45928"
ts=2026-05-06T05:35:10.93723387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55760"
ts=2026-05-06T05:35:21.231523781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49476"
level=debug ts=2026-05-06T05:35:22.213121525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:35:25.725976717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:35:55.729018257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:36:05.103420792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33148"
ts=2026-05-06T05:36:10.941758436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60042"
level=debug ts=2026-05-06T05:36:16.691078184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:36:21.237469655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40686"
ts=2026-05-06T05:36:25.73152707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:36:55.733626649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:37:05.110192083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45850"
level=debug ts=2026-05-06T05:37:17.580986861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:37:17.582926419Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:37:25.737101959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:37:35.113575923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:37426"
ts=2026-05-06T05:37:55.74022028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=error ts=2026-05-06T05:38:09.931899243Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:38:09.934475066Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:38:09.937338108Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:38:09.937381425Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-06T05:38:15.808624637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:38:25.743032092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:38:51.25107644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39974"
ts=2026-05-06T05:38:55.745408837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:39:02.129314854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:39:05.119710248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:39262"
ts=2026-05-06T05:39:10.958246542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49072"
ts=2026-05-06T05:39:25.747581713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:39:35.122870986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:38584"
level=debug ts=2026-05-06T05:39:45.187675857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:39:55.750562824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:40:25.752506552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:40:42.064972462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:40:51.26300992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33888"
ts=2026-05-06T05:40:55.754957612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:41:25.757135034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:41:35.899975744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:41:35.901555233Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:41:55.758666924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:42:21.269065633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41340"
level=debug ts=2026-05-06T05:42:22.375929822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:42:25.760887559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:42:35.139412982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:32948"
ts=2026-05-06T05:42:55.763281311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:43:19.634580051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:43:25.76532871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:43:35.144931731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:59732"
ts=2026-05-06T05:43:40.983412895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42424"
ts=2026-05-06T05:43:51.275741241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35696"
ts=2026-05-06T05:43:55.767040258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:44:07.631285595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:44:25.769460131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:44:55.771277886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:45:07.698826543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:45:10.99015357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:54184"
ts=2026-05-06T05:45:25.773536031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:45:51.311819254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:45:51.313775559Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:45:55.77583581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:46:05.151659154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:58868"
ts=2026-05-06T05:46:25.778063839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:46:35.154754022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:41938"
level=debug ts=2026-05-06T05:46:36.091159012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:46:55.780941576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T05:47:17.873077947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:47:21.293290634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42566"
ts=2026-05-06T05:47:25.78391049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:47:55.786555206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:48:05.162296634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43336"
level=error ts=2026-05-06T05:48:09.929320505Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:48:09.931157584Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:48:09.933061424Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:48:09.933086266Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:48:11.005791168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47970"
level=debug ts=2026-05-06T05:48:11.161305383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:48:25.789511151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:48:55.791949501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:48:57.117509562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:49:25.793145957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:49:41.012952452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:37184"
level=debug ts=2026-05-06T05:49:43.104204339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:49:43.106077962Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:49:51.305467426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52976"
ts=2026-05-06T05:49:55.79479111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:50:25.797541816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T05:50:33.848901899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:50:55.800240729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T05:51:23.501344802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:51:25.803306598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:51:35.174171184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:55818"
ts=2026-05-06T05:51:41.022847604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45220"
ts=2026-05-06T05:51:55.805107828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:52:11.024080847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34418"
level=debug ts=2026-05-06T05:52:19.369950474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:52:25.807557308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:52:35.178202771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46652"
ts=2026-05-06T05:52:55.809730953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:53:09.339695635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:53:21.324838963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38492"
ts=2026-05-06T05:53:25.812252914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:53:41.032502177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:56680"
level=debug ts=2026-05-06T05:53:54.654562442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:53:54.656267475Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:53:55.814673636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:54:05.187141389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:33048"
ts=2026-05-06T05:54:25.817303512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T05:54:35.190276774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:40292"
level=debug ts=2026-05-06T05:54:35.947278966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:54:41.035989474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40784"
ts=2026-05-06T05:54:51.330675317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53724"
ts=2026-05-06T05:54:55.818847604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:55:25.820317392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:55:26.688384238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:55:55.822574913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:56:05.195721912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:39836"
ts=2026-05-06T05:56:11.043187465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55904"
level=debug ts=2026-05-06T05:56:18.649454606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:56:25.82428692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:56:41.044673569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36722"
ts=2026-05-06T05:56:51.339176893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42244"
ts=2026-05-06T05:56:55.826286274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T05:57:01.922048697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:57:11.045970814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39702"
ts=2026-05-06T05:57:25.827969744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:57:55.829152024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:57:58.612158459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T05:57:58.613962774Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:58:09.930336235Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:58:09.932175363Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:58:09.9340534Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T05:58:09.934074809Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T05:58:25.830423143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T05:58:35.207054081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43666"
level=debug ts=2026-05-06T05:58:49.186068768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:58:55.832715446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T05:59:25.835027053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T05:59:34.92493173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T05:59:35.211337154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:47762"
ts=2026-05-06T05:59:51.354200308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38904"
ts=2026-05-06T05:59:55.836835362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T06:00:05.214663546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45154"
level=debug ts=2026-05-06T06:00:20.72089487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:00:21.357194976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53334"
ts=2026-05-06T06:00:25.840193267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:00:55.842882403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T06:01:05.218922145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:56048"
ts=2026-05-06T06:01:21.361585573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33426"
level=debug ts=2026-05-06T06:01:21.995089794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:01:25.846242515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:01:41.07025752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58746"
ts=2026-05-06T06:01:51.364137904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54922"
ts=2026-05-06T06:01:55.849263705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T06:02:00.655219632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T06:02:00.657675357Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:02:25.852110337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:02:41.074021332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:46268"
ts=2026-05-06T06:02:55.854787726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T06:02:57.668652959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:03:05.226712972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:39968"
ts=2026-05-06T06:03:11.075157539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:51244"
ts=2026-05-06T06:03:25.857176729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T06:03:46.139956777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:03:55.859841843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:04:25.862200294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T06:04:35.232096694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:58846"
level=debug ts=2026-05-06T06:04:46.766303604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:04:55.864387063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:05:05.234837042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:49906"
ts=2026-05-06T06:05:21.378555524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39744"
ts=2026-05-06T06:05:25.866044159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T06:05:35.237226639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57740"
level=debug ts=2026-05-06T06:05:43.523560099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:05:55.866942877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T06:06:11.091496849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57028"
ts=2026-05-06T06:06:21.382443765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58680"
ts=2026-05-06T06:06:25.869351036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T06:06:35.243174344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:55036"
level=debug ts=2026-05-06T06:06:35.778599439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T06:06:35.780186587Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:06:51.385581359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36502"
ts=2026-05-06T06:06:55.871775378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T06:07:05.245713292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:39578"
ts=2026-05-06T06:07:11.095325143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38386"
ts=2026-05-06T06:07:21.389066485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43362"
ts=2026-05-06T06:07:25.873592579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T06:07:34.183557415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:07:35.24905301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45950"
ts=2026-05-06T06:07:51.392238724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40944"
ts=2026-05-06T06:07:55.875976576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=error ts=2026-05-06T06:08:09.93066542Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:08:09.932445014Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:08:09.934084104Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:08:09.93410573Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:08:21.394940369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55682"
ts=2026-05-06T06:08:25.877989032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T06:08:31.820827143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:08:35.254736779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:35376"
ts=2026-05-06T06:08:51.398408386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58726"
ts=2026-05-06T06:08:55.878726891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T06:09:05.257438294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:57896"
level=debug ts=2026-05-06T06:09:09.428103476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:09:25.880312472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T06:09:55.882099303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T06:10:00.387033904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:10:21.405001218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43406"
ts=2026-05-06T06:10:25.884635604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T06:10:55.88748921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T06:11:01.771699391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T06:11:01.773270643Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:11:05.265382967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:36574"
ts=2026-05-06T06:11:25.8898536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T06:11:35.26785705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:60934"
ts=2026-05-06T06:11:41.120566445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58414"
level=debug ts=2026-05-06T06:11:50.718633449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:11:51.412680577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50778"
ts=2026-05-06T06:11:55.891984488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:12:21.416219978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52572"
ts=2026-05-06T06:12:25.895084912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T06:12:25.975631015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:12:55.89736504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T06:13:05.274699308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:36686"
ts=2026-05-06T06:13:11.127583241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50992"
level=debug ts=2026-05-06T06:13:24.763397009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:13:25.898912534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
ts=2026-05-06T06:13:41.128430534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55000"
ts=2026-05-06T06:13:55.900829721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=debug ts=2026-05-06T06:14:04.415546361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:14:25.904187089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:14:41.131776615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33666"
level=debug ts=2026-05-06T06:14:50.994027537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T06:14:50.995677754Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:14:51.426418855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60044"
ts=2026-05-06T06:14:55.906647468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:15:21.429375814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55588"
ts=2026-05-06T06:15:25.909562013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T06:15:29.392005785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:15:55.912417569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T06:16:15.824353551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:16:25.915303601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:16:41.141277071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:32906"
ts=2026-05-06T06:16:51.435098328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55898"
level=debug ts=2026-05-06T06:16:52.855877644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:16:55.917668851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
ts=2026-05-06T06:17:25.919184948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-67e391d4 10.130.0.61:7946"
level=debug ts=2026-05-06T06:17:44.174881234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T06:17:55.921780293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vkdt9-d0da1438 10.129.0.67:7946"
level=error ts=2026-05-06T06:18:09.930225858Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:18:09.932128331Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:18:09.934208431Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T06:18:09.934228792Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:18:11.147283324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58190"
ts=2026-05-06T06:18:21.442925777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39234"
ts=2026-05-06T06:18:25.924313579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
level=debug ts=2026-05-06T06:18:39.072299796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T06:18:39.074084108Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T06:18:41.149114032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55548"
ts=2026-05-06T06:18:55.92708661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-56c6z-c6900c65 10.129.0.66:7946"
ts=2026-05-06T06:19:05.310844761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:40002"
ts=2026-05-06T06:19:11.150874295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44162"