level=info ts=2026-04-29T14:07:07.263360741Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-29T14:07:07.263395733Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-29T14:07:07.266080035Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-29T14:07:07.266555413Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T14:07:07.266666936Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.68 inf=eth0
level=debug ts=2026-04-29T14:07:07.267761872Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-29T14:07:07.267848932Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.68 inf=eth0
level=info ts=2026-04-29T14:07:07.268191663Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-3dac6b50
ts=2026-04-29T14:07:07.268263412Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-29T14:07:07.268276218Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.68 advertisePort=7946
level=debug ts=2026-04-29T14:07:07.268299426Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.68 advertisePort=7946
level=info ts=2026-04-29T14:07:07.268432776Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-29T14:07:07.275267474Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T14:07:07.275287212Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-29T14:07:07.275301241Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.870611ms
level=error ts=2026-04-29T14:07:07.275313338Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-29T14:07:07.275329923Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-29T14:07:07.278273716Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T14:07:07.27828847Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T14:07:07.285438474Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-29T14:07:07.285453532Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-29T14:07:07.285474607Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-29T14:07:07.285484178Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-29T14:07:07.285479984Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-29T14:07:07.285524632Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-29T14:07:07.285547596Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-29T14:07:07.285622972Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-29T14:07:07.28563843Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-29T14:07:07.286055704Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-29T14:07:07.286122012Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-29T14:07:07.286169855Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-29T14:07:07.286186121Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-29T14:07:07.286341602Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-29T14:07:07.286361162Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-29T14:07:07.286382315Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-29T14:07:08.286637094Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-29T14:07:08.430896769Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-29T14:07:08.430942752Z caller=loki.go:615 msg="Loki started" startup_time=1.241985506s
ts=2026-04-29T14:07:09.066647854Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T14:07:09.066675942Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:07:12.10036034Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T14:07:12.10038846Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-29T14:07:13.431078383Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-29T14:07:13.431131Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-29T14:07:18.772540548Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-29T14:07:18.772564654Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-29T14:07:30.739873115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:07:31.254459361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.70:7946"
ts=2026-04-29T14:07:31.255743764Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
ts=2026-04-29T14:07:31.256547997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
ts=2026-04-29T14:07:31.257498168Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.66:7946: dial tcp 10.129.0.66:7946: connect: connection refused"
ts=2026-04-29T14:07:31.258134591Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.64:7946: dial tcp 10.129.0.64:7946: connect: connection refused"
level=info ts=2026-04-29T14:07:31.258143869Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=23.982808325s
ts=2026-04-29T14:07:41.738951084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50220"
level=debug ts=2026-04-29T14:07:50.420513275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:07:51.16945038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:07:56.360111408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46560"
ts=2026-04-29T14:08:04.052934765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51174"
level=debug ts=2026-04-29T14:08:05.307438637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:08:21.171001442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:08:27.13419802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:08:51.172661149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:08:52.200913928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59030"
level=debug ts=2026-04-29T14:08:57.213524462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T14:08:57.215432605Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:09:21.174763425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:09:26.299585353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:09:51.176172289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:10:21.177826589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:10:22.206351745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49612"
level=debug ts=2026-04-29T14:10:26.573526615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:10:51.179759316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:11:06.82531844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53056"
ts=2026-04-29T14:11:21.180798255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T14:11:25.661402724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:11:36.827718049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:36216"
ts=2026-04-29T14:11:51.182457166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:11:52.21197406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53234"
ts=2026-04-29T14:12:04.06669722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55220"
ts=2026-04-29T14:12:06.829821439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53762"
ts=2026-04-29T14:12:21.18365292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:12:22.238484017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-29T14:12:51.185244573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:12:52.214197745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53692"
ts=2026-04-29T14:13:06.83443454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34382"
ts=2026-04-29T14:13:21.187254586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:13:30.83215409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-29T14:13:30.833671625Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:13:51.188694752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T14:14:17.692728813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:14:21.189630096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:14:22.218920726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39682"
ts=2026-04-29T14:14:51.19125034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T14:15:09.54000315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:15:21.19308811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:15:22.221725175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37404"
ts=2026-04-29T14:15:51.194715639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T14:15:58.989610392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:16:06.847354028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46574"
ts=2026-04-29T14:16:21.19621351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:16:34.083160743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34666"
ts=2026-04-29T14:16:36.849202254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38442"
ts=2026-04-29T14:16:51.197502184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:16:55.76333478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-29T14:17:13.432137031Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-29T14:17:13.435791621Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:17:13.437723134Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:17:13.439617154Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:17:13.439638636Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-29T14:17:13.439650092Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-04-29T14:17:21.199333169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:17:22.227922211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44100"
level=debug ts=2026-04-29T14:17:34.153218327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:17:34.154789719Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:17:51.2009841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:18:06.855616083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53618"
ts=2026-04-29T14:18:21.203211034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:18:31.126218655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:18:36.857312993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:48054"
ts=2026-04-29T14:18:51.205003366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:19:04.091922344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47876"
level=debug ts=2026-04-29T14:19:21.103355794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:19:21.206896426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:19:51.208375891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T14:20:02.534239245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:20:21.209897209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:20:22.239466514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51904"
level=debug ts=2026-04-29T14:20:38.825197024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:20:51.211757638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:21:06.86674417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51302"
ts=2026-04-29T14:21:21.2128828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:21:34.099642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43148"
level=debug ts=2026-04-29T14:21:34.810445467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:21:34.811941426Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:21:36.868437752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:37872"
ts=2026-04-29T14:21:51.214350411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:22:04.100926225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51088"
level=debug ts=2026-04-29T14:22:16.046864709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:22:21.215741129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:22:51.216966238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:22:54.562231849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:23:21.217914002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:23:36.876829434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59098"
level=debug ts=2026-04-29T14:23:51.068780087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:23:51.21942011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:23:52.252203453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33344"
ts=2026-04-29T14:24:06.879087505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:39294"
ts=2026-04-29T14:24:21.221142169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:24:34.110617982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:59158"
ts=2026-04-29T14:24:51.222827563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:24:52.937769836Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:25:21.224327022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:25:32.699647046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:25:32.701127233Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:25:34.113620871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48862"
ts=2026-04-29T14:25:36.884818892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:39190"
ts=2026-04-29T14:25:51.225637198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:25:52.256739117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36930"
level=debug ts=2026-04-29T14:26:09.137393075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:26:21.227512688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:26:51.060188524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:26:51.228931329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=error ts=2026-04-29T14:27:13.445322033Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:27:13.447372177Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:27:13.44966014Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:27:13.449682819Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:27:21.229922586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:27:41.973337376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:27:51.231456824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:28:06.895627512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:49610"
level=debug ts=2026-04-29T14:28:19.46089134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:28:21.233116192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:28:51.234801623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:29:12.810849851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:29:12.812422085Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:29:21.236645272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:29:22.268924918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57418"
ts=2026-04-29T14:29:51.238787058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:30:04.128029068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34974"
ts=2026-04-29T14:30:06.904300209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:58140"
level=debug ts=2026-04-29T14:30:13.370219412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:30:21.240118706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:30:22.273336609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45992"
ts=2026-04-29T14:30:34.129317405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52224"
ts=2026-04-29T14:30:36.906186503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50922"
ts=2026-04-29T14:30:51.241697373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:30:52.274398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48618"
level=debug ts=2026-04-29T14:30:54.846566712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:31:04.131206011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42596"
ts=2026-04-29T14:31:21.242576079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:31:51.24411429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T14:31:55.532010069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:32:21.246188914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:32:51.24840072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:32:55.788354148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:33:21.250819303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:33:22.283411934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41020"
ts=2026-04-29T14:33:51.252776189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:33:52.285081249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58292"
level=debug ts=2026-04-29T14:33:56.02189732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:33:56.024118945Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:34:21.254714266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:34:22.286306089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60676"
ts=2026-04-29T14:34:34.141974762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35506"
ts=2026-04-29T14:34:36.921425719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53338"
ts=2026-04-29T14:34:51.256783908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:34:51.810898626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:35:21.258292224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:35:36.925376705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:39378"
level=debug ts=2026-04-29T14:35:50.569710884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:35:51.259794327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:35:52.290786338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34386"
ts=2026-04-29T14:36:06.927805394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:58300"
ts=2026-04-29T14:36:21.260927226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:36:22.292131797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37546"
ts=2026-04-29T14:36:51.262387432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:36:51.330214321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:37:04.151953301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43014"
ts=2026-04-29T14:37:06.931309101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53182"
level=error ts=2026-04-29T14:37:13.443317121Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:37:13.445011995Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:37:13.446602905Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:37:13.446625587Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:37:21.263729867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:37:39.857694214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:37:51.265996322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:38:04.15377379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52620"
ts=2026-04-29T14:38:21.267549535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:38:22.299210489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36938"
level=debug ts=2026-04-29T14:38:35.683177328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:38:35.684609836Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:38:36.93715539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:58086"
ts=2026-04-29T14:38:51.268935129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:38:52.30086759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42320"
level=debug ts=2026-04-29T14:39:10.213973791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:39:21.271118243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:39:34.157410524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37528"
level=debug ts=2026-04-29T14:39:50.984175283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:39:51.273205214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:39:52.303929115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34348"
ts=2026-04-29T14:40:06.944142044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45178"
ts=2026-04-29T14:40:21.274777636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:40:22.305751888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53818"
level=debug ts=2026-04-29T14:40:42.918964193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:40:51.281222244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:41:21.28429785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:41:31.806298632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:41:51.285553202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:42:21.287374314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:42:22.311877912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39250"
level=debug ts=2026-04-29T14:42:33.113277857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:42:33.115498168Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:42:34.187982054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35746"
ts=2026-04-29T14:42:36.961178058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:52960"
ts=2026-04-29T14:42:51.288838618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:42:52.313081306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60766"
ts=2026-04-29T14:43:06.962584168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45170"
ts=2026-04-29T14:43:21.290698619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:43:31.92569858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:43:51.292233248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:43:52.319199986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40452"
ts=2026-04-29T14:44:21.293480905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:44:22.320896795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45778"
level=debug ts=2026-04-29T14:44:29.689900492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:44:34.196793894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:54726"
ts=2026-04-29T14:44:51.29494484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:44:52.321935481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40488"
ts=2026-04-29T14:45:21.296634169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:45:22.322903339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54072"
level=debug ts=2026-04-29T14:45:32.118995194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:45:51.298711278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:46:04.201754539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56374"
ts=2026-04-29T14:46:06.979295191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47730"
ts=2026-04-29T14:46:21.300493678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:46:22.326272793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52030"
level=debug ts=2026-04-29T14:46:26.477168092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:46:36.981142421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51596"
ts=2026-04-29T14:46:51.302157928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:46:52.327569604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57632"
ts=2026-04-29T14:47:04.205773777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47666"
ts=2026-04-29T14:47:06.98300697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:57410"
level=error ts=2026-04-29T14:47:13.445776684Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:47:13.447879194Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:47:13.449868446Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:47:13.449899324Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:47:21.303506994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:47:21.449531788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:47:21.451061754Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:47:22.328600955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52082"
ts=2026-04-29T14:47:51.304847056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:48:04.209372668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45010"
level=debug ts=2026-04-29T14:48:20.042663322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:48:21.306654833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:48:51.3083792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:49:00.567850107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:49:06.991006592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55958"
ts=2026-04-29T14:49:21.30936617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:49:47.496365454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:49:51.311455547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:50:06.995363729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51500"
ts=2026-04-29T14:50:21.312767142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:50:22.340477538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52396"
level=debug ts=2026-04-29T14:50:36.49902934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:50:51.314642754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:50:52.342144859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55742"
ts=2026-04-29T14:51:06.999360501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50172"
ts=2026-04-29T14:51:21.316329447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:51:22.343863238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:43664"
level=debug ts=2026-04-29T14:51:34.44049407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:51:34.442418483Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:51:37.001808872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56866"
ts=2026-04-29T14:51:51.317809841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:52:04.222256984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60346"
ts=2026-04-29T14:52:07.004214025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38176"
level=debug ts=2026-04-29T14:52:16.634776367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:52:21.319488465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:52:37.005865379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:37548"
ts=2026-04-29T14:52:51.321960344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:52:52.348794851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52438"
level=debug ts=2026-04-29T14:53:13.034218875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:53:21.323966615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:53:51.325765624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:53:52.351938812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50520"
level=debug ts=2026-04-29T14:53:58.506624484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:54:04.229089221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52592"
ts=2026-04-29T14:54:21.327401905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T14:54:48.848644637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:54:51.330741364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T14:55:21.332884922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:55:22.356434205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53034"
ts=2026-04-29T14:55:37.019574312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:48822"
level=debug ts=2026-04-29T14:55:42.705592788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:55:42.707168485Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:55:51.334518992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T14:55:52.357301851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54086"
ts=2026-04-29T14:56:07.021528817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59352"
ts=2026-04-29T14:56:21.33699754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:56:34.509709464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:56:37.02330987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:40336"
ts=2026-04-29T14:56:51.338473433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=error ts=2026-04-29T14:57:13.44377292Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:57:13.445694036Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:57:13.447385892Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T14:57:13.44740829Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:57:21.339876491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T14:57:21.591908886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:57:51.341928039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:58:04.243649992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43432"
level=debug ts=2026-04-29T14:58:08.998394087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:58:21.343675471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:58:22.38046888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55424"
ts=2026-04-29T14:58:51.345585849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:58:52.382840498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33814"
level=debug ts=2026-04-29T14:58:58.946991893Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T14:59:04.246689318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56450"
ts=2026-04-29T14:59:21.348013914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T14:59:22.384670305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35896"
ts=2026-04-29T14:59:37.035653932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:39830"
level=debug ts=2026-04-29T14:59:39.171303532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T14:59:39.173247134Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T14:59:51.349875929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:00:21.351549346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:00:22.387727749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39846"
level=debug ts=2026-04-29T15:00:36.800952814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:00:51.353024715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T15:01:17.807211189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:01:21.354828234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:01:51.355705683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T15:02:04.792055067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:02:21.35741014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:02:22.394162968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41352"
ts=2026-04-29T15:02:51.359801542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T15:03:00.659204181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:03:07.050524852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:42976"
ts=2026-04-29T15:03:21.361586114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:03:34.264656989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:32994"
ts=2026-04-29T15:03:37.052612097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44510"
level=debug ts=2026-04-29T15:03:41.197524033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:03:41.199391888Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:03:51.362920943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:04:04.26676073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38018"
ts=2026-04-29T15:04:07.0545695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53114"
ts=2026-04-29T15:04:21.364393009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T15:04:41.46950294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:04:51.366465038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:04:52.403984989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52706"
level=debug ts=2026-04-29T15:05:17.692368502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:05:21.367938838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:05:34.273357779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44300"
ts=2026-04-29T15:05:51.36929671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:05:52.40729238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48092"
level=debug ts=2026-04-29T15:05:52.417382946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:06:04.274344866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38412"
ts=2026-04-29T15:06:21.37121171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T15:06:28.135524829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:06:51.37456224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=error ts=2026-04-29T15:07:13.442202932Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:07:13.443987389Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:07:13.445605057Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:07:13.445626451Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-29T15:07:13.982864663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:07:13.984584509Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:07:21.376355212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:07:34.279294135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:43008"
ts=2026-04-29T15:07:51.377887267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:08:04.281115723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48866"
level=debug ts=2026-04-29T15:08:12.383186018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:08:21.379911007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:08:34.282639515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50910"
ts=2026-04-29T15:08:51.380733439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T15:09:04.83994394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:09:21.381841482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:09:22.419980515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49044"
ts=2026-04-29T15:09:34.286347389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37672"
level=debug ts=2026-04-29T15:09:41.245083742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:09:51.383157402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:10:21.385108603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:10:22.422642566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50646"
ts=2026-04-29T15:10:34.289701551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50826"
level=debug ts=2026-04-29T15:10:35.631698046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:10:51.386713569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:11:21.38850859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T15:11:29.640436708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:11:29.641933204Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:11:51.389786484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:12:21.391635329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T15:12:21.461759181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:12:51.393980257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:12:52.430169153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37478"
level=debug ts=2026-04-29T15:13:01.042067899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:13:04.299175587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57550"
ts=2026-04-29T15:13:21.395712639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:13:34.300186164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35746"
ts=2026-04-29T15:13:51.397660091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:13:52.433784068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59238"
level=debug ts=2026-04-29T15:13:58.300272561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:14:04.301635963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47364"
ts=2026-04-29T15:14:07.094339552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34454"
ts=2026-04-29T15:14:21.398671206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:14:34.302702338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:51528"
ts=2026-04-29T15:14:51.399640271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:14:52.43673783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55354"
level=debug ts=2026-04-29T15:14:58.766981568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:15:07.098667503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:39962"
ts=2026-04-29T15:15:21.401503257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:15:22.43811131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33226"
ts=2026-04-29T15:15:34.305774079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50028"
ts=2026-04-29T15:15:37.100713916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59146"
level=debug ts=2026-04-29T15:15:47.958155435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:15:47.959853156Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:15:51.403324385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:16:21.40532547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T15:16:23.921212317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:16:37.105155913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:37826"
ts=2026-04-29T15:16:51.406524582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:17:04.310467702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50090"
ts=2026-04-29T15:17:07.106842872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45090"
level=error ts=2026-04-29T15:17:13.444262954Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:17:13.446140588Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:17:13.44791218Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:17:13.44793936Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:17:21.408261986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T15:17:25.075614732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:17:51.410147454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
level=debug ts=2026-04-29T15:18:05.004523601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:18:07.111164441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:41802"
ts=2026-04-29T15:18:21.411341615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
ts=2026-04-29T15:18:34.314534489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56346"
ts=2026-04-29T15:18:37.113062401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47268"
level=debug ts=2026-04-29T15:18:49.39091006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:18:51.412840828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:19:21.414023898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:19:37.116724392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34174"
level=debug ts=2026-04-29T15:19:38.8763776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:19:38.878096978Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:19:51.415595758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:20:04.318800274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60204"
ts=2026-04-29T15:20:21.417126339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-6q7pc-ca93f9e4 10.128.0.70:7946"
level=debug ts=2026-04-29T15:20:37.419880679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:20:51.419182349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:21:04.321957098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55924"
ts=2026-04-29T15:21:07.122191349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:60668"
level=debug ts=2026-04-29T15:21:13.043281147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:21:21.420126956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:21:22.45800253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38316"
ts=2026-04-29T15:21:51.421624367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:21:52.459499583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50204"
level=debug ts=2026-04-29T15:22:10.175408572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:22:21.423281833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:22:22.460978691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46876"
ts=2026-04-29T15:22:34.326545335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58246"
ts=2026-04-29T15:22:37.127813582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56282"
ts=2026-04-29T15:22:51.424725018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:22:52.462204551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55760"
level=debug ts=2026-04-29T15:22:59.431593144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:23:21.425803566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T15:23:40.652708421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:23:40.654191026Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:23:51.427177682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:23:52.464689419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50612"
ts=2026-04-29T15:24:04.331466022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60506"
ts=2026-04-29T15:24:07.132950197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51558"
ts=2026-04-29T15:24:21.428716557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:24:34.332909958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38066"
ts=2026-04-29T15:24:37.135164177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59878"
level=debug ts=2026-04-29T15:24:40.304650788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:24:51.430201324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:25:07.137225088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53592"
ts=2026-04-29T15:25:21.431221671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T15:25:28.24870213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:25:37.139176106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59902"
ts=2026-04-29T15:25:51.432467396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T15:26:03.237455193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-29T15:26:21.433938475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
ts=2026-04-29T15:26:51.435128885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-sd7h9-0304c0f6 10.129.0.67:7946"
level=debug ts=2026-04-29T15:27:01.463353187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:27:13.443770892Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:27:13.445586208Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:27:13.447270397Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-29T15:27:13.447294057Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:27:21.436864181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"
ts=2026-04-29T15:27:37.147945517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:50742"
level=debug ts=2026-04-29T15:27:50.197647132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-29T15:27:50.199257847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-29T15:27:51.438335983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-2c180afa 10.129.0.69:7946"