level=info ts=2026-04-20T12:12:27.759609228Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T12:12:27.759651419Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T12:12:27.760720569Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T12:12:27.761029893Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T12:12:27.761100391Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.64 inf=eth0
level=debug ts=2026-04-20T12:12:27.761969323Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T12:12:27.762046602Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.64 inf=eth0
level=info ts=2026-04-20T12:12:27.762108544Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-9ff68b3e
ts=2026-04-20T12:12:27.762164628Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T12:12:27.762178761Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.64 advertisePort=7946
level=debug ts=2026-04-20T12:12:27.762198061Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.64 advertisePort=7946
level=info ts=2026-04-20T12:12:27.762317014Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T12:12:27.768445585Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:12:27.768466212Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:12:27.768479986Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.165218ms
level=error ts=2026-04-20T12:12:27.768490651Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T12:12:27.768504621Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T12:12:27.771655558Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:12:27.771674636Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:12:27.776286298Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-20T12:12:27.776310308Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T12:12:27.776317822Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-20T12:12:27.776341664Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-20T12:12:27.77638591Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-20T12:12:27.776417749Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-20T12:12:27.776435505Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-20T12:12:27.776451853Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-20T12:12:27.776680983Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-20T12:12:27.776691049Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T12:12:27.776745409Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T12:12:27.776746048Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T12:12:27.776790113Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T12:12:27.776927643Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T12:12:27.901357296Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T12:12:27.901439376Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T12:12:28.902396954Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T12:12:28.902448561Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T12:12:28.902474948Z caller=loki.go:599 msg="Loki started" startup_time=1.217874279s
ts=2026-04-20T12:12:29.674180184Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:12:29.67420272Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:12:32.731319019Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:12:32.731343753Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T12:12:33.902854196Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T12:12:33.90289649Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T12:12:39.770037651Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:12:39.770060838Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:12:45.944942743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:12:50.749413429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:44846"
ts=2026-04-20T12:12:51.654080039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:55994"
ts=2026-04-20T12:12:53.962543756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.64:7946"
ts=2026-04-20T12:12:53.962647617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:37688"
ts=2026-04-20T12:12:53.963098792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.62:7946"
ts=2026-04-20T12:12:53.965015296Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.105:7946: dial tcp 10.128.0.105:7946: connect: connection refused"
ts=2026-04-20T12:12:53.965936244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.103:7946"
ts=2026-04-20T12:12:53.968125481Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.102:7946: dial tcp 10.128.0.102:7946: connect: connection refused"
ts=2026-04-20T12:12:53.969882471Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.68:7946: dial tcp 10.130.0.68:7946: connect: connection refused"
level=info ts=2026-04-20T12:12:53.96989303Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.201382292s
ts=2026-04-20T12:12:54.248498667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:55944"
ts=2026-04-20T12:13:01.131170167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:43946"
level=debug ts=2026-04-20T12:13:07.672578754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:13:07.967072726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:43666"
ts=2026-04-20T12:13:09.057084869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:13:30.670595304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:13:39.059199334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:13:51.747826507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:14:07.969305103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:36592"
ts=2026-04-20T12:14:09.062144256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:14:22.398693916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T12:14:22.400377068Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:14:25.969650527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:47246"
ts=2026-04-20T12:14:37.970744699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:53346"
ts=2026-04-20T12:14:39.064368291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:15:01.142260605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:46410"
ts=2026-04-20T12:15:07.97152907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:57818"
ts=2026-04-20T12:15:09.065470422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:15:10.531791409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:15:37.97288268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:43794"
ts=2026-04-20T12:15:39.066137135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:15:55.975538368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:53484"
ts=2026-04-20T12:16:01.147903539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:56014"
ts=2026-04-20T12:16:09.067724619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:16:16.412224553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:16:39.070346865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:16:55.978310866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57866"
ts=2026-04-20T12:17:09.072961112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:17:14.298939247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:17:25.979860863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:53748"
ts=2026-04-20T12:17:39.075881079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:18:09.077982568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:18:10.465220639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T12:18:31.161950098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:49436"
ts=2026-04-20T12:18:39.079531587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:19:09.08113389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:19:16.430827353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T12:19:16.432514986Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:19:25.986779036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:42788"
ts=2026-04-20T12:19:31.168117353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:38120"
ts=2026-04-20T12:19:39.083228518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:20:07.99417156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:35710"
ts=2026-04-20T12:20:09.084816338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:20:13.782880146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:20:39.085619228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:21:09.087053136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:21:11.131731132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:21:38.000325046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:41892"
ts=2026-04-20T12:21:39.088084598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:21:51.965656025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:22:01.182848993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:51272"
ts=2026-04-20T12:22:08.001611858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:49538"
ts=2026-04-20T12:22:09.090125624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:22:25.996660101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:33630"
level=debug ts=2026-04-20T12:22:30.301368653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:22:33.903271102Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T12:22:33.905714547Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:22:33.907586623Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:22:33.909495714Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:22:33.909519899Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:22:33.909531836Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-20T12:22:39.092573861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:23:01.188978989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:44016"
ts=2026-04-20T12:23:09.095441756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:23:20.752416045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:23:20.753885672Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:23:26.000978331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:50462"
ts=2026-04-20T12:23:39.097857967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:24:08.006172438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:48266"
ts=2026-04-20T12:24:09.100128811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:24:10.131944955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:24:31.198143816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:45230"
ts=2026-04-20T12:24:39.102226061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:24:56.005573966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:39512"
level=debug ts=2026-04-20T12:25:07.773943116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:25:08.008632733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:40232"
ts=2026-04-20T12:25:09.104145229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:25:39.105619446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:25:48.559154469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:26:09.107159992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:26:31.504392142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:26:39.108827043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:27:01.211599054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:35314"
ts=2026-04-20T12:27:09.109923985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:27:14.898700465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:27:14.900304571Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:27:31.214211715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:55732"
ts=2026-04-20T12:27:39.111895254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:28:04.014623001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:28:09.11330568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:28:39.11494312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:28:56.020605912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48674"
ts=2026-04-20T12:29:01.222499549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:41354"
level=debug ts=2026-04-20T12:29:01.767146882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:29:09.116906486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:29:26.022299603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:37932"
ts=2026-04-20T12:29:31.225705446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:55894"
ts=2026-04-20T12:29:38.028893731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:47782"
ts=2026-04-20T12:29:39.119298587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:29:47.51614789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:30:09.121860095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:30:26.018355198Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:30:26.026950198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:41430"
ts=2026-04-20T12:30:39.123539429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:30:56.028283099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:38662"
ts=2026-04-20T12:31:08.034107937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:35226"
ts=2026-04-20T12:31:09.124642255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:31:23.122042141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:31:23.124141354Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:31:31.237080358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:41198"
ts=2026-04-20T12:31:38.036095225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:34804"
ts=2026-04-20T12:31:39.12604011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:31:56.032795822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57608"
level=debug ts=2026-04-20T12:31:59.976139242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:32:01.244069146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:38284"
ts=2026-04-20T12:32:08.037870086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:56054"
ts=2026-04-20T12:32:09.127662147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=error ts=2026-04-20T12:32:33.912577979Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:32:33.914220008Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:32:33.91584303Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:32:33.915873509Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:32:38.039347848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:41178"
ts=2026-04-20T12:32:39.128423028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:32:55.976962963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:33:01.257651748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:46962"
ts=2026-04-20T12:33:08.040647617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:37642"
ts=2026-04-20T12:33:09.130450216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:33:31.26030299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:39638"
ts=2026-04-20T12:33:38.042020505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:37750"
ts=2026-04-20T12:33:39.134880196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:33:41.516381366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:34:08.043244795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:57912"
ts=2026-04-20T12:34:09.136661746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:34:23.040724144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:34:39.137939442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:34:56.042409708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:41810"
level=debug ts=2026-04-20T12:35:02.253396126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:35:02.25486657Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:35:09.140330391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:35:26.043975658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:50576"
ts=2026-04-20T12:35:31.272258131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:52608"
ts=2026-04-20T12:35:39.143579379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:35:56.044998266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:50798"
level=debug ts=2026-04-20T12:35:57.531826787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:36:01.274579679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:43556"
ts=2026-04-20T12:36:09.14591527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:36:38.050753765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:47380"
ts=2026-04-20T12:36:39.147382792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:36:49.357130548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:36:56.048589649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:49772"
ts=2026-04-20T12:37:09.148952513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:37:26.049713455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:33488"
level=debug ts=2026-04-20T12:37:35.462895289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:37:38.053275509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:47068"
ts=2026-04-20T12:37:39.150015947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:37:56.050593444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48110"
ts=2026-04-20T12:38:09.154189514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:38:15.133520371Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:38:31.289484328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:53958"
ts=2026-04-20T12:38:38.056373773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:36968"
ts=2026-04-20T12:38:39.155193842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:39:01.272938171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:39:01.27465769Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:39:08.058077706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:38960"
ts=2026-04-20T12:39:09.156033008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:39:31.294629301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:41184"
ts=2026-04-20T12:39:39.158287853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:39:51.73477847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:40:09.160365994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:40:31.300367395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:39764"
ts=2026-04-20T12:40:39.161636129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:40:53.593347836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:40:56.061986284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:60336"
ts=2026-04-20T12:41:01.303537486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:39124"
ts=2026-04-20T12:41:08.065661319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:51026"
ts=2026-04-20T12:41:09.163079533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:41:39.164129165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:41:41.262988304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:42:09.165721084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:42:22.791547505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:42:33.912979527Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:42:33.914849122Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:42:33.916483028Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:42:33.91651541Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:42:39.168867102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:43:01.317856033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:37104"
ts=2026-04-20T12:43:09.171101651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:43:22.699937754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:43:22.701740873Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:43:39.173043858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:43:56.073183053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48700"
level=debug ts=2026-04-20T12:44:08.595790971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:44:09.174146218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:44:26.07441331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:60404"
ts=2026-04-20T12:44:39.174831589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:44:59.689322009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:45:01.329482259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:53932"
ts=2026-04-20T12:45:08.081200892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:44356"
ts=2026-04-20T12:45:09.176192492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:45:26.076407339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:42092"
ts=2026-04-20T12:45:38.082316868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:44960"
ts=2026-04-20T12:45:39.177006861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:45:41.822339177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:45:56.07796859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48856"
ts=2026-04-20T12:46:01.340342004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:43740"
ts=2026-04-20T12:46:09.178322732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:46:25.811066999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:46:31.343380086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:33452"
ts=2026-04-20T12:46:38.08663376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:43734"
ts=2026-04-20T12:46:39.179219251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:47:08.087678266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:38328"
ts=2026-04-20T12:47:09.180843991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:47:26.916038996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:47:26.917665378Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:47:39.183290448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:48:07.356369259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:48:09.186364447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:48:26.090502547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:53508"
ts=2026-04-20T12:48:31.354378073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:43234"
ts=2026-04-20T12:48:39.188454115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:48:53.945786693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:49:09.189907062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:49:39.191639826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:49:50.056345907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:49:56.093975351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:38198"
ts=2026-04-20T12:50:09.193829449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:50:36.229973814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:50:39.194841647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:51:08.108463065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:47028"
ts=2026-04-20T12:51:09.196512644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:51:35.344818208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:51:35.346584809Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:51:39.198753846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:52:09.201049221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=error ts=2026-04-20T12:52:33.913732411Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:52:33.91551334Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:52:33.917049023Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T12:52:33.917079608Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:52:36.4726032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:52:38.112787053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:44766"
ts=2026-04-20T12:52:39.202918389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:53:01.381566666Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:41114"
ts=2026-04-20T12:53:09.205165002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:53:26.110541294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57926"
level=debug ts=2026-04-20T12:53:29.380706647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:53:39.206799025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:54:01.388393753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:44386"
ts=2026-04-20T12:54:08.118966275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:37942"
ts=2026-04-20T12:54:09.209275038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:54:24.514743633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:54:39.211035193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
ts=2026-04-20T12:55:01.393812746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:42498"
level=debug ts=2026-04-20T12:55:06.27072012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:55:09.2125057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:55:39.213201193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:55:53.281351939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:55:53.284711271Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:56:01.40033131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:57774"
ts=2026-04-20T12:56:09.214435988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:56:26.121461521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:40248"
level=debug ts=2026-04-20T12:56:34.178475189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:56:39.216471102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:57:09.218339409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:57:12.373388915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:57:31.409644806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:53432"
ts=2026-04-20T12:57:39.219421379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:57:47.634830584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:58:08.137081119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:50854"
ts=2026-04-20T12:58:09.22176883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:58:26.127521829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:36258"
ts=2026-04-20T12:58:31.415304815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:44612"
ts=2026-04-20T12:58:38.138371018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:45244"
ts=2026-04-20T12:58:39.223427009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:58:43.110514718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:59:09.224398433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:59:19.093862062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:59:19.095525375Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T12:59:26.132439301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:43938"
ts=2026-04-20T12:59:31.420271928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:35784"
ts=2026-04-20T12:59:38.142025873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:53552"
ts=2026-04-20T12:59:39.226099752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d370e8cb 10.129.0.65:7946"
level=debug ts=2026-04-20T12:59:59.85610074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:00:01.423107435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:42108"
ts=2026-04-20T13:00:08.143290434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:33282"
ts=2026-04-20T13:00:09.22698553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T13:00:39.229655154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T13:00:51.687259815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:01:08.1490428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:50384"
ts=2026-04-20T13:01:09.232467243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T13:01:26.059021771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T13:01:31.431908118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:47004"
ts=2026-04-20T13:01:38.150732404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:47014"
ts=2026-04-20T13:01:39.234796044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"