level=info ts=2026-04-24T22:41:51.702868815Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-24T22:41:51.702901139Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-24T22:41:51.704270052Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-24T22:41:51.704724074Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-24T22:41:51.704824883Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.59 inf=eth0
level=debug ts=2026-04-24T22:41:51.705943239Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-24T22:41:51.706026699Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.59 inf=eth0
level=info ts=2026-04-24T22:41:51.706114081Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-d48aaaeb
ts=2026-04-24T22:41:51.706195084Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-24T22:41:51.706212094Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.59 advertisePort=7946
level=debug ts=2026-04-24T22:41:51.706231561Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.59 advertisePort=7946
level=info ts=2026-04-24T22:41:51.706412889Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-24T22:41:51.718248431Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-24T22:41:51.718280569Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-24T22:41:51.718294316Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=11.88347ms
level=error ts=2026-04-24T22:41:51.718304084Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-24T22:41:51.71831416Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-24T22:41:51.721911681Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T22:41:51.721926548Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-24T22:41:51.724455417Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-24T22:41:51.724563917Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-24T22:41:51.724674765Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-24T22:41:51.72470244Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-24T22:41:51.72475769Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-24T22:41:51.724858422Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-24T22:41:51.724875194Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-24T22:41:51.7249551Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-24T22:41:51.72496531Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-24T22:41:51.724970836Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-24T22:41:51.725011049Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-24T22:41:51.725051832Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-24T22:41:51.725071811Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-24T22:41:51.72523342Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-24T22:41:51.725241882Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-24T22:41:51.725255933Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-24T22:41:52.726081299Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-24T22:41:52.882360134Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-24T22:41:52.882418413Z caller=loki.go:599 msg="Loki started" startup_time=1.25921246s
ts=2026-04-24T22:41:53.371206359Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T22:41:53.371231574Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:41:56.100792748Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T22:41:56.100818149Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-24T22:41:57.883063832Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-24T22:41:57.88311304Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-24T22:42:01.4284804Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T22:42:01.42851506Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:42:11.876957592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.92:7946"
ts=2026-04-24T22:42:11.879493531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.91:7946"
ts=2026-04-24T22:42:11.881965389Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.90:7946: dial tcp 10.128.0.90:7946: connect: connection refused"
ts=2026-04-24T22:42:11.884425659Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.83:7946: dial tcp 10.129.0.83:7946: connect: connection refused"
level=info ts=2026-04-24T22:42:11.884438859Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=20.166101208s
level=debug ts=2026-04-24T22:42:12.996768159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:42:25.657513266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:47558"
level=debug ts=2026-04-24T22:42:26.374648291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:42:41.665067425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T22:42:47.77735859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:43:11.667178593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T22:43:15.03686392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:43:41.670684906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T22:43:42.925646258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-24T22:43:42.927265443Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:43:53.057753195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:50040"
ts=2026-04-24T22:44:01.863985283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:45086"
ts=2026-04-24T22:44:11.673630764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T22:44:17.609174692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:44:23.061188323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48400"
ts=2026-04-24T22:44:25.667122009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:57414"
ts=2026-04-24T22:44:41.676493024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T22:44:53.064541677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:35588"
ts=2026-04-24T22:45:11.679515887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T22:45:21.868570655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:45:23.067612129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40192"
ts=2026-04-24T22:45:41.681452484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
ts=2026-04-24T22:46:01.874242501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:50788"
ts=2026-04-24T22:46:11.683613554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T22:46:17.549127073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:46:41.686123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T22:47:11.689749316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T22:47:17.423725862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T22:47:41.692640179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T22:47:53.076901807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:43278"
ts=2026-04-24T22:48:11.695585342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T22:48:23.079781124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41916"
level=debug ts=2026-04-24T22:48:23.688295422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-24T22:48:23.689907657Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:48:41.697979286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T22:49:01.887676246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47756"
ts=2026-04-24T22:49:11.699515754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T22:49:16.590854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:49:41.701516517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T22:49:55.700837465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:56400"
ts=2026-04-24T22:50:01.894194211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:40678"
level=debug ts=2026-04-24T22:50:03.367002977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:50:11.704046415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
ts=2026-04-24T22:50:23.087705067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:53254"
ts=2026-04-24T22:50:31.897448555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:36638"
ts=2026-04-24T22:50:41.705981732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T22:50:43.193404866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:51:11.708132054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
ts=2026-04-24T22:51:41.709511479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T22:51:44.32976572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-24T22:51:57.883936121Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-24T22:51:57.887211017Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:51:57.888939216Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:51:57.890640087Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:51:57.890668011Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-24T22:51:57.890681387Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-24T22:52:11.711820567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T22:52:38.614786411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:52:38.616532734Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:52:41.714984157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T22:53:11.718453965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T22:53:29.554595638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:53:41.721307812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T22:53:53.101964589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44328"
ts=2026-04-24T22:53:55.721518689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:49028"
level=debug ts=2026-04-24T22:54:07.827398957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:54:11.724076933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T22:54:23.105428507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33788"
ts=2026-04-24T22:54:25.723060067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:51240"
ts=2026-04-24T22:54:31.914188209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:34156"
ts=2026-04-24T22:54:41.726981369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T22:54:53.108916798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:35826"
level=debug ts=2026-04-24T22:55:09.822643752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:55:11.729452827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T22:55:31.918114523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:44094"
ts=2026-04-24T22:55:41.732652296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T22:55:44.854300984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:55:55.730601655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:38152"
ts=2026-04-24T22:56:11.734757095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T22:56:19.845896967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:56:19.847663983Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:56:23.114706215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41756"
ts=2026-04-24T22:56:41.736754358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T22:57:01.92379251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:54338"
ts=2026-04-24T22:57:11.738584365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T22:57:14.818375847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:57:23.121006956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:57062"
ts=2026-04-24T22:57:41.739832972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
ts=2026-04-24T22:58:11.741034713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T22:58:17.03822442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:58:25.744148817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:55856"
ts=2026-04-24T22:58:31.933015936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:44892"
ts=2026-04-24T22:58:41.742933744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T22:59:11.746195105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T22:59:15.619635921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:59:41.749756058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:00:11.751777072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T23:00:12.923674182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:00:25.754609221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:56866"
ts=2026-04-24T23:00:41.7531904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T23:01:01.468345837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:01:01.470423976Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:01:11.755528457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:01:25.758836859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:45748"
ts=2026-04-24T23:01:31.946406067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58676"
ts=2026-04-24T23:01:41.758189561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:01:42.257279227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:01:57.902338528Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:01:57.905506184Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:01:57.908262112Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:01:57.908300568Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:02:11.761494771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:02:23.141699876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:46140"
ts=2026-04-24T23:02:25.763580374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39594"
ts=2026-04-24T23:02:31.95108514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:35002"
level=debug ts=2026-04-24T23:02:32.34980092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:02:41.764778405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:03:11.767718598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:03:31.9285583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:03:41.770031236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
ts=2026-04-24T23:04:11.772515257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:04:17.211219789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:04:25.774375397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:47288"
ts=2026-04-24T23:04:41.77550406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:04:58.683831495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:04:58.685855452Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:05:01.961789091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:51742"
ts=2026-04-24T23:05:11.777613348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
ts=2026-04-24T23:05:31.964907048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55666"
level=debug ts=2026-04-24T23:05:39.726508645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:05:41.778972077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
ts=2026-04-24T23:05:53.157421655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44612"
ts=2026-04-24T23:06:11.780757142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T23:06:20.171689997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:06:23.160506965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:38370"
ts=2026-04-24T23:06:31.968873149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53108"
ts=2026-04-24T23:06:41.783316475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T23:07:00.034906702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:07:11.784506438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
ts=2026-04-24T23:07:23.166384044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:49492"
ts=2026-04-24T23:07:31.972892532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:57856"
ts=2026-04-24T23:07:41.789710559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T23:07:50.352880871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:08:01.975685769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53052"
ts=2026-04-24T23:08:11.800688379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:08:31.981227927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:51754"
ts=2026-04-24T23:08:41.80396575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:08:45.676290918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:08:45.678220472Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:08:53.17619008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:39902"
ts=2026-04-24T23:09:01.984337915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:34874"
ts=2026-04-24T23:09:11.806249884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-329eab8b 10.130.0.60:7946"
level=debug ts=2026-04-24T23:09:31.853408887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:09:41.813232085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:10:07.430992324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:10:11.821387203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:10:23.184016958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:42980"
ts=2026-04-24T23:10:41.824768919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:10:58.963163166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:11:11.828360066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:11:24.462413173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:41174"
ts=2026-04-24T23:11:24.472166775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:41176"
ts=2026-04-24T23:11:33.706727919Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-329eab8b (timeout reached)"
ts=2026-04-24T23:11:36.706600661Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-329eab8b has failed, no acks received"
ts=2026-04-24T23:11:41.831536994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:11:51.130349827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:11:57.894749831Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:11:57.897347028Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:11:57.899395251Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:11:57.899440415Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:12:02.007806425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:56434"
ts=2026-04-24T23:12:11.83341244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f4c55ddb 10.130.0.60:7946"
level=debug ts=2026-04-24T23:12:40.792495305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:12:40.795518019Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:12:41.835478044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:13:11.837644337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f4c55ddb 10.130.0.60:7946"
level=debug ts=2026-04-24T23:13:17.25121244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:13:23.199090361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:47670"
ts=2026-04-24T23:13:39.713142168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:55702"
ts=2026-04-24T23:13:41.838965522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f4c55ddb 10.130.0.60:7946"
ts=2026-04-24T23:13:53.201891232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:45654"
ts=2026-04-24T23:14:02.014416306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:34066"
level=debug ts=2026-04-24T23:14:02.728924482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:14:11.841570606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:14:41.845643143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:14:52.284337005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:14:53.205951812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:39824"
ts=2026-04-24T23:15:11.847705664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f4c55ddb 10.130.0.60:7946"
ts=2026-04-24T23:15:32.020284515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:34074"
level=debug ts=2026-04-24T23:15:34.421047602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:15:41.850188323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:15:53.212495746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:58672"
ts=2026-04-24T23:16:11.852978204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f4c55ddb 10.130.0.60:7946"
ts=2026-04-24T23:16:23.215782072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:35356"
level=debug ts=2026-04-24T23:16:32.779838439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:16:32.781720679Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:16:39.731662486Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39984"
ts=2026-04-24T23:16:41.85423532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f4c55ddb 10.130.0.60:7946"
ts=2026-04-24T23:16:53.219000864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44988"
ts=2026-04-24T23:17:11.855291415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f4c55ddb 10.130.0.60:7946"
ts=2026-04-24T23:17:23.222475391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41598"
level=debug ts=2026-04-24T23:17:29.938188552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:17:39.736944012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:42696"
ts=2026-04-24T23:17:41.856687242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f4c55ddb 10.130.0.60:7946"
ts=2026-04-24T23:17:53.225947772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44114"
ts=2026-04-24T23:18:11.858776059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:18:23.228934474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:42608"
level=debug ts=2026-04-24T23:18:28.290386084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:18:28.707758295Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f4c55ddb (timeout reached)"
ts=2026-04-24T23:18:31.707679836Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f4c55ddb has failed, no acks received"
ts=2026-04-24T23:18:32.033756117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43184"
ts=2026-04-24T23:18:40.593455565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40176"
ts=2026-04-24T23:18:40.604398748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40178"
ts=2026-04-24T23:18:41.860563702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118356fa 10.130.0.60:7946"
ts=2026-04-24T23:18:43.708180947Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f4c55ddb (timeout reached)"
ts=2026-04-24T23:18:46.708082737Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f4c55ddb has failed, no acks received"
ts=2026-04-24T23:18:51.708444151Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-f4c55ddb as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T23:19:11.862856032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:19:23.985567081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:19:39.955405968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:37028"
ts=2026-04-24T23:19:41.866364441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:20:00.632626622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:20:11.869711967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:20:23.237232557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51202"
level=debug ts=2026-04-24T23:20:38.520137191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:20:38.521769625Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:20:41.872551956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:20:53.240781837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:53018"
ts=2026-04-24T23:21:02.047225527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:51224"
ts=2026-04-24T23:21:09.962641837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:41738"
ts=2026-04-24T23:21:11.875626041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:21:23.244297972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44552"
level=debug ts=2026-04-24T23:21:39.440362525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:21:41.877736331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118356fa 10.130.0.60:7946"
level=error ts=2026-04-24T23:21:57.895674715Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:21:57.897929569Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:21:57.900080583Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:21:57.900112074Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:22:11.879478548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118356fa 10.130.0.60:7946"
level=debug ts=2026-04-24T23:22:18.345896066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:22:41.882058981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:23:08.458959106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:23:09.972043103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:33334"
ts=2026-04-24T23:23:11.885216736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:23:32.059416001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:45032"
ts=2026-04-24T23:23:41.888735252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:24:02.062238277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46454"
ts=2026-04-24T23:24:09.976022011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:37772"
level=debug ts=2026-04-24T23:24:10.247137527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:24:11.890891619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118356fa 10.130.0.60:7946"
ts=2026-04-24T23:24:39.977620161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:50580"
ts=2026-04-24T23:24:41.892964548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:24:53.260476528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:46388"
level=debug ts=2026-04-24T23:24:55.612524439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:24:55.614719141Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:25:11.895618434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-118356fa 10.130.0.60:7946"
level=debug ts=2026-04-24T23:25:38.067952049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:25:41.908505805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:26:09.985590595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:36132"
ts=2026-04-24T23:26:11.914509384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:26:28.979316478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:26:32.078831563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:37710"
ts=2026-04-24T23:26:41.917879069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:26:53.710387946Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-118356fa (timeout reached)"
ts=2026-04-24T23:26:56.70943064Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-118356fa has failed, no acks received"
ts=2026-04-24T23:26:58.710541612Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-118356fa (timeout reached)"
ts=2026-04-24T23:27:01.710347852Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-118356fa has failed, no acks received"
ts=2026-04-24T23:27:02.081640834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39092"
ts=2026-04-24T23:27:07.620545034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:60832"
ts=2026-04-24T23:27:07.630752173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:60844"
ts=2026-04-24T23:27:11.920974365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:27:16.70961135Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-118356fa as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-24T23:27:24.304735141Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:27:41.92292227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
level=debug ts=2026-04-24T23:27:59.774637818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:28:02.08737335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39482"
ts=2026-04-24T23:28:11.925457456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:28:41.927943898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:28:54.67961607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:28:54.681279222Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:29:11.930256176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:29:23.279797143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:54474"
level=debug ts=2026-04-24T23:29:41.494611682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:29:41.933406625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:30:11.935183576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
level=debug ts=2026-04-24T23:30:25.725051033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:30:41.93602232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:30:44.545706739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:33390"
ts=2026-04-24T23:30:53.288515387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44112"
ts=2026-04-24T23:31:11.938061464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:31:13.485548182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:31:14.54701294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:46286"
ts=2026-04-24T23:31:23.291856034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:59822"
ts=2026-04-24T23:31:41.940882862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:31:57.366806577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:31:57.893252006Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:31:57.894934029Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:31:57.896499558Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:31:57.896521835Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:32:02.105556719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46338"
ts=2026-04-24T23:32:11.943098723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:32:41.944993903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:32:46.477806432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:32:46.479257442Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:33:11.946807472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:33:32.113406638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39726"
level=debug ts=2026-04-24T23:33:40.560485721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:33:41.949056524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:33:53.303186446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:52874"
ts=2026-04-24T23:34:11.952745617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:34:20.481961429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:34:23.305615015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48724"
ts=2026-04-24T23:34:32.118518153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:54936"
ts=2026-04-24T23:34:41.955130814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:34:53.308983627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:36954"
ts=2026-04-24T23:35:02.121221108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:36862"
ts=2026-04-24T23:35:11.957405331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:35:17.873736694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:35:41.960033715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:35:44.570111565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:55210"
level=debug ts=2026-04-24T23:36:11.060236691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:36:11.962664976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:36:14.571797161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:57918"
ts=2026-04-24T23:36:23.315849943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:42792"
ts=2026-04-24T23:36:32.13121078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:36872"
ts=2026-04-24T23:36:41.964809166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:36:53.319109281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:57826"
level=debug ts=2026-04-24T23:36:53.332443032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:36:53.334046797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:37:02.13480738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:40234"
ts=2026-04-24T23:37:11.967402012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:37:41.970374555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:37:44.578531146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40906"
level=debug ts=2026-04-24T23:37:50.214978188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:37:53.32553366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33632"
ts=2026-04-24T23:38:02.141539807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48446"
ts=2026-04-24T23:38:11.973655374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:38:14.580314238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:37394"
ts=2026-04-24T23:38:41.976626876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:38:48.735944155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:38:53.329706542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44370"
ts=2026-04-24T23:39:11.978810494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:39:23.333122696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44200"
level=debug ts=2026-04-24T23:39:33.097091505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:39:41.980669217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:40:02.151608247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58234"
ts=2026-04-24T23:40:11.983066646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:40:23.338240424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:55462"
level=debug ts=2026-04-24T23:40:32.97078472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:40:41.98569826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:40:44.594191148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:42706"
level=debug ts=2026-04-24T23:41:11.145283106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:41:11.146973542Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:41:11.988721599Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:41:23.342759275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:52154"
ts=2026-04-24T23:41:32.156923955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:48458"
ts=2026-04-24T23:41:41.992319334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:41:44.59865961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:43324"
level=debug ts=2026-04-24T23:41:53.066666839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:41:57.893363919Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:41:57.895552218Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:41:57.89725394Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:41:57.897278984Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:42:11.995072823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:42:14.600138659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:48496"
ts=2026-04-24T23:42:41.998670498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:42:48.611601595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:43:12.001086982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c2f45ad5 10.130.0.60:7946"
ts=2026-04-24T23:43:14.603970781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:43144"
ts=2026-04-24T23:43:23.352663863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41054"
ts=2026-04-24T23:43:31.882064412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:49702"
ts=2026-04-24T23:43:31.891687481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:49718"
ts=2026-04-24T23:43:33.707956903Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c2f45ad5 (timeout reached)"
level=debug ts=2026-04-24T23:43:35.483464967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:43:36.707183991Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c2f45ad5 has failed, no acks received"
ts=2026-04-24T23:43:42.00317612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:43:53.355979436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:43306"
ts=2026-04-24T23:43:56.707438591Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-c2f45ad5 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T23:43:58.708497227Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-c2f45ad5 (timeout reached)"
ts=2026-04-24T23:44:01.707786834Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-c2f45ad5 has failed, no acks received"
ts=2026-04-24T23:44:12.005768108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:44:23.358535992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:37782"
ts=2026-04-24T23:44:24.75040824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:55110"
ts=2026-04-24T23:44:32.171741805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:57924"
level=debug ts=2026-04-24T23:44:35.451039581Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:44:42.007735959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:44:53.360989911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33676"
ts=2026-04-24T23:44:54.751819304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:38056"
ts=2026-04-24T23:45:12.009295969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-24T23:45:15.660123406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:45:15.661777727Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:45:23.364118514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:49338"
ts=2026-04-24T23:45:42.010795138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-24T23:46:07.657488579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:46:12.012931333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:46:42.015920053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:46:54.761147225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:42856"
level=debug ts=2026-04-24T23:47:00.193480767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:47:02.180504613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:45058"
ts=2026-04-24T23:47:12.018867143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:47:24.762533853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39706"
ts=2026-04-24T23:47:42.022285866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:47:42.438693631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:48:12.02558769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:48:19.040971996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:48:42.028244105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:49:12.030040075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-24T23:49:20.496910015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:49:20.498627276Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:49:23.377859246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:53912"
ts=2026-04-24T23:49:24.773225607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:52896"
ts=2026-04-24T23:49:42.031778425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:49:54.774602688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:41396"
level=debug ts=2026-04-24T23:50:08.763776446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:50:12.032767465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:50:32.19365527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:45472"
ts=2026-04-24T23:50:42.034873498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:50:53.386761473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:40706"
ts=2026-04-24T23:50:54.778473756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:43476"
level=debug ts=2026-04-24T23:50:57.32447326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:51:02.196568694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55216"
ts=2026-04-24T23:51:12.037848452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:51:32.199487416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:52506"
ts=2026-04-24T23:51:42.041027548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:51:53.39292279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:49584"
level=error ts=2026-04-24T23:51:57.892839215Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:51:57.894422377Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:51:57.895879362Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:51:57.895902157Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-24T23:51:59.041591365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:52:12.044526197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:52:23.396252995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:38478"
level=debug ts=2026-04-24T23:52:37.971014016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:52:42.046546592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:52:53.39955651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33188"
ts=2026-04-24T23:53:12.048525302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:53:35.549769756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:53:35.551539478Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:53:42.051659071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:54:12.054851982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:54:24.096561948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:54:32.211075938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:41738"
ts=2026-04-24T23:54:42.057306063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:54:53.408222007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51694"
ts=2026-04-24T23:55:12.059435662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:55:14.516933982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:55:23.410786128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:38606"
ts=2026-04-24T23:55:42.062643514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:56:12.065807972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-24T23:56:13.073618498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:56:42.069354454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-24T23:56:54.811798886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:42168"
level=debug ts=2026-04-24T23:57:03.69985535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:57:12.072560272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-24T23:57:24.813477569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:53500"
ts=2026-04-24T23:57:32.229411224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:38590"
ts=2026-04-24T23:57:42.075440934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-24T23:57:51.62281793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:57:51.624833309Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:58:12.077754289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:58:23.42368849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51784"
ts=2026-04-24T23:58:32.23558126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:46264"
level=debug ts=2026-04-24T23:58:32.259530257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:58:42.079001104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:59:12.080561597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:59:23.429706102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:51764"
level=debug ts=2026-04-24T23:59:23.594747914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:59:32.23985521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:40248"
ts=2026-04-24T23:59:42.081903033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-24T23:59:54.826892282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:37324"
level=debug ts=2026-04-24T23:59:58.62742389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:00:02.242607582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53590"
ts=2026-04-25T00:00:12.083800193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:00:42.086359315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:00:51.487720104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:00:54.831572437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:51674"
ts=2026-04-25T00:01:02.248525336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:53010"
ts=2026-04-25T00:01:12.088781331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-25T00:01:28.443035106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:01:28.44495338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:01:42.091907117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-25T00:01:53.439364516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:39988"
level=error ts=2026-04-25T00:01:57.894900036Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:01:57.896557888Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:01:57.898151386Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:01:57.898178977Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-25T00:02:05.076726388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:02:12.095064261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:02:24.838642053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:57964"
ts=2026-04-25T00:02:42.096911854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:02:57.752948802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:03:12.099067574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:03:23.446316178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41626"
ts=2026-04-25T00:03:24.843518454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:50408"
ts=2026-04-25T00:03:42.102063981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-25T00:03:45.617311533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:03:54.844585848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:49864"
ts=2026-04-25T00:04:02.264362701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:47834"
ts=2026-04-25T00:04:12.104310342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:04:24.846270799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:49850"
ts=2026-04-25T00:04:32.266950312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:35708"
level=debug ts=2026-04-25T00:04:37.935436172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:04:42.106118813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-25T00:04:53.45596973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:48520"
ts=2026-04-25T00:04:54.847723877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:45948"
ts=2026-04-25T00:05:12.108239245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:05:23.836256941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:05:23.837861763Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:05:42.109411818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:06:12.111044673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:06:19.518129857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:06:23.461570595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:60336"
ts=2026-04-25T00:06:24.855337189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:56602"
ts=2026-04-25T00:06:32.275283055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:57390"
ts=2026-04-25T00:06:42.113104979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:06:55.641773225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:07:12.115087508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:07:32.278915489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:59186"
ts=2026-04-25T00:07:42.118300558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:07:53.46723313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:41198"
level=debug ts=2026-04-25T00:07:53.601452442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:08:02.281800341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:58770"
ts=2026-04-25T00:08:12.121392542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:08:42.124866416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-25T00:08:53.473652854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:33530"
level=debug ts=2026-04-25T00:08:53.798307638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:09:12.12721953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:09:24.872047402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:41846"
level=debug ts=2026-04-25T00:09:29.707714583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:09:29.709386817Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:09:32.288901085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43044"
ts=2026-04-25T00:09:42.12934498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:09:53.480094367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:45788"
ts=2026-04-25T00:10:12.131769715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:10:24.876419361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:52956"
level=debug ts=2026-04-25T00:10:30.659728092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:10:42.133444873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:11:02.295484592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:42816"
level=debug ts=2026-04-25T00:11:05.492047144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:11:12.13528438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:11:42.138008159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=error ts=2026-04-25T00:11:57.893868973Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:11:57.89569653Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:11:57.897381057Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-25T00:11:57.897407145Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-25T00:11:59.427747986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:12:12.140579435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:12:40.422517017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:12:42.141784899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:13:12.143544458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:13:23.496674772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:52950"
ts=2026-04-25T00:13:24.892763944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:49262"
level=debug ts=2026-04-25T00:13:36.398085186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:13:36.400133928Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:13:42.145827424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:13:53.499540381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:44744"
ts=2026-04-25T00:13:54.894025028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:39142"
level=debug ts=2026-04-25T00:14:10.821589136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:14:12.14772117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:14:23.502747057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:59608"
ts=2026-04-25T00:14:42.150556651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-25T00:14:49.936443611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:14:54.898753843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:48886"
ts=2026-04-25T00:15:12.153504867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:15:23.507481245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:56844"
ts=2026-04-25T00:15:32.312442938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:51418"
ts=2026-04-25T00:15:42.156747517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
level=debug ts=2026-04-25T00:15:47.461788307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:15:54.903373121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:52232"
ts=2026-04-25T00:16:12.160349451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:16:32.316466474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39062"
level=debug ts=2026-04-25T00:16:33.681703687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:16:42.16331263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
ts=2026-04-25T00:17:02.31982129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:38560"
level=debug ts=2026-04-25T00:17:08.77320997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:17:08.775087214Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:17:12.166835978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-25T00:17:32.322653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:57834"
ts=2026-04-25T00:17:42.168994349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:17:54.914110876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:48642"
level=debug ts=2026-04-25T00:18:04.589898559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:18:12.170368655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:18:42.171295583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:18:53.735698802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:19:02.330246149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:43806"
ts=2026-04-25T00:19:12.172862047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
ts=2026-04-25T00:19:42.174849283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-5x5zd-22da33a3 10.128.0.92:7946"
level=debug ts=2026-04-25T00:19:47.584076886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:20:02.336782505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:44456"
ts=2026-04-25T00:20:12.176953217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:20:25.27979105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-25T00:20:32.340010249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:55384"
ts=2026-04-25T00:20:42.178589235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f6da1526 10.130.0.60:7946"
level=debug ts=2026-04-25T00:21:03.198024459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-25T00:21:03.199679349Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-25T00:21:12.180527382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-25T00:21:23.536942254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.91:37858"
ts=2026-04-25T00:21:32.344521724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.92:39008"
ts=2026-04-25T00:21:42.183637072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-2jmm6-3542e305 10.128.0.91:7946"
ts=2026-04-25T00:21:43.405091439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40868"
ts=2026-04-25T00:21:43.415300509Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.60:40878"