level=info ts=2026-05-19T02:40:08.608303313Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-19T02:40:08.608333094Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-19T02:40:08.609485311Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-19T02:40:08.6097849Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-19T02:40:08.609858673Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.79 inf=eth0
level=debug ts=2026-05-19T02:40:08.610552568Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-19T02:40:08.610609611Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.79 inf=eth0
level=info ts=2026-05-19T02:40:08.610691215Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-74723448
ts=2026-05-19T02:40:08.610759703Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-19T02:40:08.610773752Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.79 advertisePort=7946
level=debug ts=2026-05-19T02:40:08.610797177Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.79 advertisePort=7946
level=info ts=2026-05-19T02:40:08.610931597Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-19T02:40:08.61880811Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T02:40:08.618827736Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-19T02:40:08.618840189Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.910169ms
level=error ts=2026-05-19T02:40:08.61885172Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-19T02:40:08.618865076Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-19T02:40:08.621775569Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:40:08.621793737Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-19T02:40:08.627488704Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-19T02:40:08.627481848Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-05-19T02:40:08.627505702Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-19T02:40:08.627511728Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-19T02:40:08.627516119Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-19T02:40:08.62757829Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-19T02:40:08.627662913Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-19T02:40:08.627670399Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-19T02:40:08.62775906Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-19T02:40:08.627769429Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-19T02:40:08.627810987Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-19T02:40:08.62784433Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-19T02:40:08.627864358Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-19T02:40:08.628004044Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-19T02:40:08.628017321Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-19T02:40:08.628031284Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-19T02:40:09.628608561Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-19T02:40:09.760866366Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-19T02:40:09.760910712Z caller=loki.go:615 msg="Loki started" startup_time=1.217850222s
ts=2026-05-19T02:40:10.354134835Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:40:10.354162077Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:40:14.084946613Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:40:14.084967898Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-19T02:40:14.761326394Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-19T02:40:14.761374786Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-19T02:40:19.543773274Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-19T02:40:19.543795888Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:40:26.168960093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49516"
ts=2026-05-19T02:40:26.220206441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33614"
level=debug ts=2026-05-19T02:40:29.114642346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:40:30.41780095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:36026"
ts=2026-05-19T02:40:34.409742249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.77:7946"
ts=2026-05-19T02:40:34.411697824Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.80:7946: dial tcp 10.128.0.80:7946: connect: connection refused"
ts=2026-05-19T02:40:34.411815562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.79:7946"
ts=2026-05-19T02:40:34.412017688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.79:48194"
level=info ts=2026-05-19T02:40:34.414354675Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=25.795482092s
ts=2026-05-19T02:40:43.709794388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:40:45.382075043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-19T02:41:04.163113759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:41:09.113669318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41350"
ts=2026-05-19T02:41:13.712961757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T02:41:18.414571594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41806"
level=debug ts=2026-05-19T02:41:35.722927557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:41:39.115114369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41530"
ts=2026-05-19T02:41:43.715427351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T02:41:48.41836029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43704"
level=debug ts=2026-05-19T02:42:04.612418365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-19T02:42:04.614132802Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T02:42:13.717047637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T02:42:43.718072434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:42:49.810838033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:43:09.120582639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41138"
ts=2026-05-19T02:43:13.720112123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T02:43:18.429236577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56142"
ts=2026-05-19T02:43:43.722180565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T02:43:43.785952279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:43:48.433009087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51030"
ts=2026-05-19T02:44:13.722929648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T02:44:18.436981207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38558"
level=debug ts=2026-05-19T02:44:35.930036143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:44:43.723750966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T02:45:09.130280346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:40140"
ts=2026-05-19T02:45:13.72570084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:45:42.22211771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-19T02:45:43.728024325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T02:46:13.730710224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T02:46:36.949068299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39698"
ts=2026-05-19T02:46:43.732830608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:46:48.528766424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-19T02:46:48.530216365Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T02:47:13.733711383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T02:47:23.591857864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:47:39.13864499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:50388"
ts=2026-05-19T02:47:43.734914391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T02:47:48.460343979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49976"
ts=2026-05-19T02:48:09.139700836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:54134"
ts=2026-05-19T02:48:13.736610152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T02:48:25.073948898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:48:43.739015166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T02:49:13.742396306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:49:17.787294706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:49:39.148298918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:37206"
ts=2026-05-19T02:49:43.745747957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T02:50:06.964313621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39658"
ts=2026-05-19T02:50:13.747630796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:50:14.416400531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-19T02:50:14.762053439Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-19T02:50:14.764092592Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T02:50:14.765735424Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T02:50:14.767266905Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T02:50:14.767289337Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-19T02:50:14.767302215Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-19T02:50:18.476102364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54328"
ts=2026-05-19T02:50:43.750216679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:51:00.491415537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T02:51:00.49328372Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T02:51:06.967661983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:48110"
ts=2026-05-19T02:51:13.753977617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:51:36.914298009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:51:36.96851113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:39206"
ts=2026-05-19T02:51:39.15975155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:53620"
ts=2026-05-19T02:51:43.756520877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T02:52:13.758013444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T02:52:18.489039943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33360"
level=debug ts=2026-05-19T02:52:22.997871141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:52:43.760190131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T02:52:48.492726527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46716"
ts=2026-05-19T02:53:06.972965911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47906"
ts=2026-05-19T02:53:13.76320214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:53:21.507654104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:53:39.169799228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:56834"
ts=2026-05-19T02:53:43.766341314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:54:05.145081165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:54:13.769804291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:54:40.906684548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T02:54:40.908363389Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T02:54:43.772751032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T02:55:06.981997664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47938"
ts=2026-05-19T02:55:13.775607863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T02:55:19.406273081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:55:43.777999082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T02:55:48.514410516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33534"
level=debug ts=2026-05-19T02:56:05.651154107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:56:06.987340868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:43060"
ts=2026-05-19T02:56:09.185913151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:47112"
ts=2026-05-19T02:56:13.780152377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T02:56:18.517962426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35956"
ts=2026-05-19T02:56:36.989276533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42208"
ts=2026-05-19T02:56:43.781951413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:56:49.991908645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:57:13.782955743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T02:57:36.163825903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:57:43.784145245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T02:58:06.993327394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45906"
ts=2026-05-19T02:58:13.785149958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T02:58:18.532354882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44592"
level=debug ts=2026-05-19T02:58:18.591381108Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T02:58:18.592845413Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T02:58:36.994305957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42150"
ts=2026-05-19T02:58:39.199563734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:60260"
ts=2026-05-19T02:58:43.786569303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T02:59:06.995572315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52910"
ts=2026-05-19T02:59:09.200547326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:38302"
level=debug ts=2026-05-19T02:59:10.206421382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T02:59:13.787787271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T02:59:18.539349688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36912"
ts=2026-05-19T02:59:43.789013521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T02:59:44.883569922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:00:13.790117976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=error ts=2026-05-19T03:00:14.771719813Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:00:14.77374787Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:00:14.775456014Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:00:14.775484043Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:00:37.001617931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60504"
level=debug ts=2026-05-19T03:00:39.15319181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:00:43.793042216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:00:48.549836513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36486"
ts=2026-05-19T03:01:07.003136937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54684"
ts=2026-05-19T03:01:13.796598574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:01:15.464172385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:01:39.211134339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:56664"
ts=2026-05-19T03:01:43.799469709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:01:48.556485242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37310"
level=debug ts=2026-05-19T03:01:55.57130759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:01:55.572757973Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:02:07.005501256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45120"
ts=2026-05-19T03:02:13.801309931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:02:39.215718379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:36090"
level=debug ts=2026-05-19T03:02:43.059684012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:02:43.80413265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:02:48.563050178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56482"
ts=2026-05-19T03:03:07.010313533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34104"
ts=2026-05-19T03:03:09.216492584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:47468"
ts=2026-05-19T03:03:13.806294133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:03:18.566461382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44970"
level=debug ts=2026-05-19T03:03:43.162370635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:03:43.807764387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:03:48.570336613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38274"
ts=2026-05-19T03:04:13.810497473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:04:37.018376388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52642"
ts=2026-05-19T03:04:39.221803125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:42888"
level=debug ts=2026-05-19T03:04:40.833332917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:04:43.812480093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:05:07.019679418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:44666"
ts=2026-05-19T03:05:13.813213635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:05:15.38595774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:05:43.813805241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:05:51.190421509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:05:51.192045821Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:06:13.816559004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:06:18.586761464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53340"
ts=2026-05-19T03:06:39.229051853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:42586"
ts=2026-05-19T03:06:43.819156861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:06:44.743114349Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:07:07.02846108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40250"
ts=2026-05-19T03:07:13.820677126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:07:32.811460569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:07:39.233313608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:54738"
ts=2026-05-19T03:07:43.822022303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:08:13.824967247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:08:18.600468248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41456"
level=debug ts=2026-05-19T03:08:34.846579175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:08:37.035163591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:56322"
ts=2026-05-19T03:08:43.828579268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:09:13.830851391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:09:31.33642444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:09:43.832600795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:10:07.041378722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:46120"
ts=2026-05-19T03:10:13.834855013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=error ts=2026-05-19T03:10:14.771141361Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:10:14.77281263Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:10:14.774559439Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:10:14.774582242Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:10:24.092663099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:10:24.094138262Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:10:39.246905248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41048"
ts=2026-05-19T03:10:43.836566355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:11:13.837700594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:11:13.99444011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:11:39.248916044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:53500"
ts=2026-05-19T03:11:43.838716794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:12:08.838378394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:12:13.839665697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:12:43.840729569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:12:44.231067476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:13:13.842465016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:13:28.290195909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:13:43.843768613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:14:09.257603729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:34598"
level=debug ts=2026-05-19T03:14:10.763015226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:14:10.765203945Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:14:13.846613173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:14:37.061268833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45632"
ts=2026-05-19T03:14:39.258478456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:42688"
ts=2026-05-19T03:14:43.849182144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:14:51.237062355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:15:07.063121659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34816"
ts=2026-05-19T03:15:09.25998731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:42120"
ts=2026-05-19T03:15:13.850165161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:15:39.261323071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41824"
level=debug ts=2026-05-19T03:15:40.06396675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:15:43.851758576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:15:48.648474849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38872"
ts=2026-05-19T03:16:13.85426759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:16:18.652192707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41850"
level=debug ts=2026-05-19T03:16:30.470660833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:16:39.263653859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:60220"
ts=2026-05-19T03:16:43.8571497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:17:09.26447363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:53364"
ts=2026-05-19T03:17:13.858921638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:17:15.636059778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:17:39.266087716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:60386"
ts=2026-05-19T03:17:43.860027531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:17:48.661776242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47092"
ts=2026-05-19T03:18:07.078213005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:56034"
level=debug ts=2026-05-19T03:18:08.452947159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:18:08.454561954Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:18:13.861426224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:18:37.079995586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52192"
ts=2026-05-19T03:18:39.268572938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:52542"
ts=2026-05-19T03:18:43.864152537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:18:51.663114329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:19:13.866762835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:19:34.856629922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:19:37.082817819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42164"
ts=2026-05-19T03:19:43.867610803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:20:13.868760756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=error ts=2026-05-19T03:20:14.771512855Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:20:14.773440039Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:20:14.775304598Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:20:14.775326796Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:20:29.398787609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:20:43.871387986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:21:12.044176449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:21:13.873124337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:21:18.684330142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34308"
ts=2026-05-19T03:21:43.87489029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:22:06.007262913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:22:06.008957272Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:22:13.876048889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:22:39.283874778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:33094"
ts=2026-05-19T03:22:43.877495483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:22:51.450324459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:23:09.285803862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:33722"
ts=2026-05-19T03:23:13.880150756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:23:39.287463484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:57572"
ts=2026-05-19T03:23:43.882096203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:23:48.701150338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45672"
level=debug ts=2026-05-19T03:23:48.934807084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:24:07.103862096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:51228"
ts=2026-05-19T03:24:13.883634641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:24:37.105024386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37042"
ts=2026-05-19T03:24:43.884521536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:24:50.33951116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:25:09.29162765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:46668"
ts=2026-05-19T03:25:13.886804908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:25:28.930984625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:25:39.293293629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59700"
ts=2026-05-19T03:25:43.889132391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:26:07.110401311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:35808"
level=debug ts=2026-05-19T03:26:11.328919215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:26:11.330400702Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:26:13.891446185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:26:39.295359906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:49696"
ts=2026-05-19T03:26:43.894248825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:27:07.115002312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:59742"
level=debug ts=2026-05-19T03:27:11.375509484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:27:13.897280177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:27:18.725069329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57540"
ts=2026-05-19T03:27:37.116204264Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:44640"
ts=2026-05-19T03:27:39.297968938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39366"
ts=2026-05-19T03:27:43.89996761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:27:48.728387829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57408"
level=debug ts=2026-05-19T03:27:50.856870968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:28:09.29943136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:33870"
ts=2026-05-19T03:28:13.901151256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:28:39.300724316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:44458"
ts=2026-05-19T03:28:43.903838933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:28:44.075108171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:28:48.735954144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41172"
ts=2026-05-19T03:29:13.905882997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:29:43.25693107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:29:43.907429326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:29:48.742761615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59934"
ts=2026-05-19T03:30:09.308892161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:51894"
ts=2026-05-19T03:30:13.909014594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=error ts=2026-05-19T03:30:14.772827533Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:30:14.774927459Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:30:14.77668459Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:30:14.776708368Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:30:35.459946962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:30:35.461549523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:30:39.310649894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41368"
ts=2026-05-19T03:30:43.910709192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:30:48.749744927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53694"
ts=2026-05-19T03:31:13.91194857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:31:22.41879646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:31:39.313031789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:51788"
ts=2026-05-19T03:31:43.912649761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:31:48.756272607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53822"
ts=2026-05-19T03:32:09.314590835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:42502"
ts=2026-05-19T03:32:13.914093046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:32:24.333064329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:32:43.915615566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:33:13.917017361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:33:23.32370853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:33:37.141065808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47184"
ts=2026-05-19T03:33:43.918512449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:34:09.320815083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:34094"
ts=2026-05-19T03:34:13.919832521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:34:18.647947871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:34:43.921419831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:35:09.325514781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39062"
level=debug ts=2026-05-19T03:35:13.814009669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:35:13.815615444Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:35:13.923679542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:35:37.149719996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55014"
ts=2026-05-19T03:35:43.925910183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:35:48.475656181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:36:09.327454284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:60286"
ts=2026-05-19T03:36:13.926989895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:36:18.785787431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51896"
level=debug ts=2026-05-19T03:36:22.557735931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:36:43.928614299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:37:13.929901228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:37:17.939943354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:37:39.332905064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:48780"
ts=2026-05-19T03:37:43.932083812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:37:48.795465864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33602"
level=debug ts=2026-05-19T03:38:07.940429714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:38:13.933866098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:38:43.934914801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:39:01.019656297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:39:01.021150966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:39:07.165182653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42046"
ts=2026-05-19T03:39:13.936598274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:39:18.80498841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39112"
ts=2026-05-19T03:39:43.937675039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:39:45.614361347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:40:07.168770893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:44206"
ts=2026-05-19T03:40:13.939042363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=error ts=2026-05-19T03:40:14.771668488Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:40:14.77335704Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:40:14.77485896Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:40:14.774878803Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:40:26.063429481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:40:43.941813775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:40:48.814366989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59872"
level=debug ts=2026-05-19T03:41:09.253257256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:41:13.94404793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:41:43.945751258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:42:07.38868167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:42:13.947680175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:42:39.358746579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:37992"
ts=2026-05-19T03:42:43.950276179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:42:46.642056519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:42:46.643488562Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:43:09.360399062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:43732"
ts=2026-05-19T03:43:13.951343026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:43:18.830954604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37220"
level=debug ts=2026-05-19T03:43:41.020554861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:43:43.9541185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:43:48.834234297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56828"
ts=2026-05-19T03:44:13.955787851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:44:20.718823304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:44:43.956833099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:45:00.491964095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:45:07.189472344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:52182"
ts=2026-05-19T03:45:09.368748011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:60700"
ts=2026-05-19T03:45:13.959351213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:45:43.961365148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:45:45.240353836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:45:48.848677733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59478"
ts=2026-05-19T03:46:09.370897043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:58150"
ts=2026-05-19T03:46:13.96358811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:46:18.852499433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50512"
level=debug ts=2026-05-19T03:46:36.003088936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:46:36.004618823Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:46:43.965726656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:47:07.199248159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40558"
ts=2026-05-19T03:47:09.375706387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:51866"
ts=2026-05-19T03:47:13.968087606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:47:17.028082694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:47:37.200483787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:38714"
ts=2026-05-19T03:47:43.97076699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:47:59.215896928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:48:13.971751726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:48:37.203304194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:36738"
ts=2026-05-19T03:48:39.379953337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:34764"
ts=2026-05-19T03:48:43.97298393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:48:48.869999256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59964"
level=debug ts=2026-05-19T03:48:51.970129835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:49:07.204527335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47402"
ts=2026-05-19T03:49:13.975517323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:49:37.205699091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47626"
ts=2026-05-19T03:49:43.978891277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:49:48.877213095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60194"
level=debug ts=2026-05-19T03:49:49.225513853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:50:13.981033229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=error ts=2026-05-19T03:50:14.771046445Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:50:14.772982157Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:50:14.774666002Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T03:50:14.774692441Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-19T03:50:26.783795665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:50:26.785228079Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:50:43.983020299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:50:48.884187444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48510"
ts=2026-05-19T03:51:07.212226921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:55394"
ts=2026-05-19T03:51:09.38856386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:46866"
ts=2026-05-19T03:51:13.986646412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:51:22.081676948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:51:43.988704892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:52:07.215879763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33560"
level=debug ts=2026-05-19T03:52:12.336750147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:52:13.990319955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:52:18.893971146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39944"
ts=2026-05-19T03:52:43.991385079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:52:48.897491332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53962"
level=debug ts=2026-05-19T03:52:53.36869492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:53:13.992469228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T03:53:37.219813499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:40566"
ts=2026-05-19T03:53:43.994609271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:53:48.904925933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46958"
level=debug ts=2026-05-19T03:53:50.998794146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:54:07.221024544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:37714"
ts=2026-05-19T03:54:09.402578436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39156"
ts=2026-05-19T03:54:13.99661062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:54:18.907772548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50910"
level=debug ts=2026-05-19T03:54:39.832050421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:54:39.833512008Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:54:43.998438337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:55:13.999702204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T03:55:41.610161219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:55:44.001756975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:56:14.005608013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T03:56:37.449111839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:56:44.008506454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:56:48.923860796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41646"
ts=2026-05-19T03:57:14.01193949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:57:37.234585298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:54130"
level=debug ts=2026-05-19T03:57:39.204595032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:57:39.417392598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:32798"
ts=2026-05-19T03:57:44.013711108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:58:07.236126592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:38424"
ts=2026-05-19T03:58:14.015731791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T03:58:18.933712042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60252"
level=debug ts=2026-05-19T03:58:21.245210551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T03:58:37.237813124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34994"
ts=2026-05-19T03:58:44.018381942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T03:58:48.937601049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53338"
ts=2026-05-19T03:59:09.421985324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:35598"
ts=2026-05-19T03:59:14.01978836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T03:59:19.05932601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T03:59:19.060908423Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T03:59:44.020762051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:00:00.561897412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:00:09.427159553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:47510"
ts=2026-05-19T04:00:14.022451863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=error ts=2026-05-19T04:00:14.772029935Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T04:00:14.773918282Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T04:00:14.775840865Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T04:00:14.775891405Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T04:00:37.249087912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:48468"
ts=2026-05-19T04:00:44.023576035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:00:44.975992192Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:01:07.250918767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:42236"
ts=2026-05-19T04:01:14.024868934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T04:01:18.955491848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41420"
level=debug ts=2026-05-19T04:01:30.625963528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:01:44.02636142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:02:05.395365308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:02:14.027640788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:02:37.257441594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:43812"
ts=2026-05-19T04:02:44.029682151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T04:02:48.966353655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57938"
level=debug ts=2026-05-19T04:03:06.987156084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:03:06.98878068Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T04:03:14.032075528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:03:44.032922456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T04:03:59.683999693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:04:09.443709825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:34296"
ts=2026-05-19T04:04:14.033628411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:04:44.034655775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T04:05:02.054092616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:05:14.036787968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T04:05:37.272893836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:60460"
ts=2026-05-19T04:05:44.039448051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:05:53.072679318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:06:07.273840648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49364"
ts=2026-05-19T04:06:14.041386751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:06:18.989052995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35832"
ts=2026-05-19T04:06:39.454931907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:37826"
ts=2026-05-19T04:06:44.044077378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:06:44.51943648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:07:09.456112144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:49094"
ts=2026-05-19T04:07:14.046883851Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T04:07:37.279724135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33334"
ts=2026-05-19T04:07:44.048438864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:07:44.289608796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:07:44.291109245Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T04:07:48.999287142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48472"
ts=2026-05-19T04:08:09.458797652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:54054"
ts=2026-05-19T04:08:14.050080255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:08:42.942997424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:08:44.0515001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:08:49.005401856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42132"
ts=2026-05-19T04:09:14.05288037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=debug ts=2026-05-19T04:09:37.030425317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:09:44.053941681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T04:09:49.011166474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45842"
ts=2026-05-19T04:10:07.293319827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:49970"
ts=2026-05-19T04:10:14.055486778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
level=error ts=2026-05-19T04:10:14.771107509Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T04:10:14.773002079Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T04:10:14.774811965Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-19T04:10:14.774832645Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T04:10:19.01414677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53220"
level=debug ts=2026-05-19T04:10:24.664764634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:10:44.057385136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:10:59.984259028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:11:07.297703869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:45682"
ts=2026-05-19T04:11:09.472208327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:59886"
ts=2026-05-19T04:11:14.059727842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:11:19.020249751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48738"
ts=2026-05-19T04:11:37.299178491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34006"
ts=2026-05-19T04:11:44.061446786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:11:46.350009593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:11:46.351459074Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T04:11:49.023318534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42518"
ts=2026-05-19T04:12:07.300414954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:34046"
ts=2026-05-19T04:12:09.477458753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:55458"
ts=2026-05-19T04:12:14.064348519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
level=debug ts=2026-05-19T04:12:29.067972099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:12:44.066000592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T04:13:14.067480212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:13:28.255292427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:13:39.485576892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:41904"
ts=2026-05-19T04:13:44.068974409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:14:03.684133766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:14:07.307745187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:41008"
ts=2026-05-19T04:14:09.48629024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:39466"
ts=2026-05-19T04:14:14.070467465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:14:37.30883129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:33480"
ts=2026-05-19T04:14:44.072111602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:14:49.044054849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42888"
level=debug ts=2026-05-19T04:14:59.648590814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:15:07.309915869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:38998"
ts=2026-05-19T04:15:14.072815859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:15:39.490430189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:50728"
level=debug ts=2026-05-19T04:15:42.185047738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-19T04:15:42.186503719Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-19T04:15:44.074641139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T04:16:14.077675271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-skgck-60484ebe 10.129.0.67:7946"
ts=2026-05-19T04:16:19.054448341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49898"
level=debug ts=2026-05-19T04:16:25.871085728Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:16:39.495380057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:52336"
ts=2026-05-19T04:16:44.080158492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
ts=2026-05-19T04:16:49.057461971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39440"
ts=2026-05-19T04:17:09.49624695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:48826"
level=debug ts=2026-05-19T04:17:10.060011333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:17:14.081168775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-49d5f4bb 10.128.0.78:7946"
ts=2026-05-19T04:17:19.060519067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53590"
ts=2026-05-19T04:17:39.49765891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.78:40410"
ts=2026-05-19T04:17:44.082484052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-j7ckr-3867ac4a 10.128.0.77:7946"
level=debug ts=2026-05-19T04:17:54.948130568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-19T04:18:07.324235862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.77:47704"