level=info ts=2026-04-27T22:51:27.127162986Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-27T22:51:27.127185284Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-27T22:51:27.128434374Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-27T22:51:27.128693209Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-27T22:51:27.128760569Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.87 inf=eth0
level=debug ts=2026-04-27T22:51:27.129300671Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-27T22:51:27.129379647Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.87 inf=eth0
level=info ts=2026-04-27T22:51:27.130047535Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-8f1b8acc
ts=2026-04-27T22:51:27.130140365Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-27T22:51:27.130152236Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.87 advertisePort=7946
level=debug ts=2026-04-27T22:51:27.130163752Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.87 advertisePort=7946
level=info ts=2026-04-27T22:51:27.130265009Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-27T22:51:27.136152033Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-27T22:51:27.13616482Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-27T22:51:27.136173604Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.910241ms
level=error ts=2026-04-27T22:51:27.136180321Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-27T22:51:27.136190877Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-27T22:51:27.138653619Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-27T22:51:27.138668955Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-27T22:51:27.143630638Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-27T22:51:27.143684624Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-27T22:51:27.143710668Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-27T22:51:27.143698089Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-27T22:51:27.14373525Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-27T22:51:27.143792688Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-27T22:51:27.143795333Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-27T22:51:27.143815331Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-27T22:51:27.144055159Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-27T22:51:27.144066849Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-27T22:51:27.144123965Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-27T22:51:27.144117714Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-27T22:51:27.144135446Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-27T22:51:27.14426753Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-27T22:51:27.144278565Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-27T22:51:27.144298538Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-27T22:51:28.144906965Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-27T22:51:28.334429482Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-27T22:51:28.334480028Z caller=loki.go:599 msg="Loki started" startup_time=1.271421505s
ts=2026-04-27T22:51:29.021530572Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-27T22:51:29.021559859Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:51:31.890515887Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-27T22:51:31.890542316Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-27T22:51:33.335414109Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-27T22:51:33.335472715Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-27T22:51:36.101701849Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.51:7946: dial tcp 10.130.0.51:7946: connect: connection refused"
ts=2026-04-27T22:51:36.103159137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.50:7946"
level=info ts=2026-04-27T22:51:36.104225695Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=8.968028448s
level=debug ts=2026-04-27T22:51:43.331249234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:52:00.353463624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T22:52:04.498181181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:52:13.202451805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:39020"
level=debug ts=2026-04-27T22:52:20.001013738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:52:30.356040244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T22:52:38.990223503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:33200"
level=debug ts=2026-04-27T22:52:44.992821094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:53:00.358749815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T22:53:07.173824073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-27T22:53:07.175699746Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T22:53:17.119267569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43114"
ts=2026-04-27T22:53:30.361137402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T22:53:41.916965667Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:53:47.121710844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:32930"
ts=2026-04-27T22:54:00.363087888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T22:54:13.212110361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:39156"
ts=2026-04-27T22:54:17.123931798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43270"
ts=2026-04-27T22:54:30.364488319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T22:54:59.924646641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:55:00.36658722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T22:55:30.369138234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T22:55:43.217704718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:52296"
ts=2026-04-27T22:56:00.371481414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T22:56:04.17798865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:56:09.012971957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37952"
ts=2026-04-27T22:56:13.219146355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:37976"
ts=2026-04-27T22:56:17.134916121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:56510"
ts=2026-04-27T22:56:30.373186859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T22:57:00.37388136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T22:57:11.591622102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-27T22:57:30.375684161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T22:58:00.377510662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T22:58:04.606818774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-27T22:58:04.608618284Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T22:58:09.02634562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:54480"
ts=2026-04-27T22:58:30.378640409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T22:58:50.98201464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T22:59:00.379946061Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T22:59:13.231280138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:33380"
ts=2026-04-27T22:59:30.38103737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T22:59:42.598990969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T22:59:43.23273642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:53792"
ts=2026-04-27T23:00:00.382562658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T23:00:09.038820473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42050"
ts=2026-04-27T23:00:30.384189507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:00:37.481783237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:00:47.156340222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:34402"
ts=2026-04-27T23:01:00.386130611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T23:01:24.670343369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:01:30.388388943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=info ts=2026-04-27T23:01:33.336382312Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-27T23:01:33.33909392Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:01:33.341175159Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:01:33.342995238Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:01:33.343025502Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-27T23:01:33.34303711Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-27T23:01:39.048880615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:36290"
ts=2026-04-27T23:01:43.241186294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:56478"
ts=2026-04-27T23:02:00.390848255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T23:02:06.126418692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:02:06.128064499Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:02:30.392326771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T23:02:39.054760978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:55550"
ts=2026-04-27T23:02:43.245550525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:48870"
ts=2026-04-27T23:02:47.167198322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:59356"
ts=2026-04-27T23:03:00.394098133Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:03:04.396072404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:03:13.247225829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:46094"
ts=2026-04-27T23:03:30.395403793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T23:03:41.755625655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:04:00.398233858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:04:17.174353931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:36106"
ts=2026-04-27T23:04:30.400041616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T23:04:33.270775804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:04:39.068112229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49070"
ts=2026-04-27T23:05:00.402364046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:05:07.944540382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:05:09.071664523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59846"
ts=2026-04-27T23:05:30.405764251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:05:47.180073259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:38494"
ts=2026-04-27T23:06:00.408474796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:06:04.154735993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:06:04.156526225Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:06:30.410520183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:07:00.412247082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T23:07:04.515323179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:07:09.084083246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:39000"
ts=2026-04-27T23:07:17.18876576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:38870"
ts=2026-04-27T23:07:30.414632263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:07:43.272214407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:45184"
ts=2026-04-27T23:07:47.191296025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:37072"
ts=2026-04-27T23:08:00.417082418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:08:05.720394733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:08:17.193513683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:51142"
ts=2026-04-27T23:08:30.419111314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T23:08:47.08570528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:09:00.421169031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:09:13.282394837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:56398"
ts=2026-04-27T23:09:30.423931287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:09:33.28426266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:09:43.284453244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:60556"
ts=2026-04-27T23:09:47.20082624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39114"
ts=2026-04-27T23:10:00.427207901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:10:30.430466361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:10:30.98965365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:10:30.992405893Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:10:47.223070856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:41084"
ts=2026-04-27T23:11:00.433175879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T23:11:09.109534799Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59720"
level=debug ts=2026-04-27T23:11:10.355791484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:11:30.433903737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=error ts=2026-04-27T23:11:33.347214191Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:11:33.349548774Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:11:33.35153765Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:11:33.351567515Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:11:39.112274197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58526"
level=debug ts=2026-04-27T23:11:51.762403463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:12:00.435810858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:12:29.128932852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:12:30.438294596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:13:00.440578478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:13:13.619034418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:13:17.243414951Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:57036"
ts=2026-04-27T23:13:30.443226632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:14:00.444901325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T23:14:11.271329525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:14:11.272986982Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:14:30.446930893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:14:39.133517885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:38492"
ts=2026-04-27T23:14:47.25249879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:58728"
ts=2026-04-27T23:15:00.449213532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:15:09.136403402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:36134"
level=debug ts=2026-04-27T23:15:13.061023125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:15:17.254501037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42108"
ts=2026-04-27T23:15:30.451473245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:16:00.452495684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T23:16:09.14333679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41902"
level=debug ts=2026-04-27T23:16:10.110769325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:16:13.314858674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:56408"
ts=2026-04-27T23:16:30.454337759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:16:43.316869636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:41008"
ts=2026-04-27T23:17:00.456526368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:17:07.886523478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:17:30.459268214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:17:47.267358091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:50172"
level=debug ts=2026-04-27T23:17:48.902788383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:18:00.461700299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:18:13.322825088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:57476"
ts=2026-04-27T23:18:30.465140148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:18:33.031681332Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:18:33.033824526Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:19:00.467563632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
level=debug ts=2026-04-27T23:19:11.050486269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:19:13.325725602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:41644"
ts=2026-04-27T23:19:30.468879968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T23:19:39.166435184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42812"
level=debug ts=2026-04-27T23:19:50.840886158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:20:00.470877988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:20:28.835291865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:20:30.473072948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-45efab44 10.129.0.89:7946"
ts=2026-04-27T23:20:39.173226716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:33320"
ts=2026-04-27T23:20:54.860818548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:34870"
ts=2026-04-27T23:20:54.870211736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:34874"
ts=2026-04-27T23:21:00.475146039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:21:06.08423396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:21:09.131154374Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-45efab44 (timeout reached)"
ts=2026-04-27T23:21:09.176097245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:45112"
ts=2026-04-27T23:21:12.130675304Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-45efab44 has failed, no acks received"
ts=2026-04-27T23:21:26.851038663Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-45efab44 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-27T23:21:30.477573198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d0cfb5 10.129.0.89:7946"
level=error ts=2026-04-27T23:21:33.349117963Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:21:33.351179031Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:21:33.353173359Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:21:33.353202555Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-27T23:21:52.011802757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:21:52.013326084Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:22:00.481644131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:22:09.183124525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56008"
ts=2026-04-27T23:22:30.48382761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d0cfb5 10.129.0.89:7946"
ts=2026-04-27T23:22:39.18611199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:38316"
level=debug ts=2026-04-27T23:22:45.819063843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:22:47.296376312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:52318"
ts=2026-04-27T23:23:00.486087046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:23:17.298324877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:60954"
ts=2026-04-27T23:23:30.488940323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:23:36.883084247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:23:47.300795807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:35904"
ts=2026-04-27T23:23:52.388348346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:59742"
ts=2026-04-27T23:24:00.492042721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:24:30.494102596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d0cfb5 10.129.0.89:7946"
level=debug ts=2026-04-27T23:24:37.036190084Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:25:00.496481611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:25:09.200716879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:57998"
ts=2026-04-27T23:25:30.499239363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:25:37.338973567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:25:47.310820961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:37296"
ts=2026-04-27T23:26:00.501371591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:26:09.207116431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53030"
ts=2026-04-27T23:26:17.313153668Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39866"
level=debug ts=2026-04-27T23:26:24.748674978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:26:24.750330521Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:26:30.503159933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:26:52.404968341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:43166"
ts=2026-04-27T23:27:00.505597852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:27:12.438305289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:27:22.4067911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:42638"
ts=2026-04-27T23:27:30.507826028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:27:57.415524143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:28:00.509231163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d0cfb5 10.129.0.89:7946"
ts=2026-04-27T23:28:17.324036791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:36554"
ts=2026-04-27T23:28:30.510911306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:28:36.665248865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:28:47.326212544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:34240"
ts=2026-04-27T23:28:52.413379032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:53608"
ts=2026-04-27T23:29:00.512500374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d0cfb5 10.129.0.89:7946"
ts=2026-04-27T23:29:09.226580812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:43700"
ts=2026-04-27T23:29:17.328183806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:55258"
ts=2026-04-27T23:29:24.131442977Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a0d0cfb5 (timeout reached)"
ts=2026-04-27T23:29:27.130666857Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a0d0cfb5 has failed, no acks received"
ts=2026-04-27T23:29:30.514749542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:29:36.770994948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:33538"
ts=2026-04-27T23:29:36.779906112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:33546"
level=debug ts=2026-04-27T23:29:36.988358403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:29:44.13227801Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a0d0cfb5 (timeout reached)"
ts=2026-04-27T23:29:47.131200556Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-a0d0cfb5 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-27T23:29:47.131370507Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a0d0cfb5 has failed, no acks received"
ts=2026-04-27T23:30:00.518105418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55924244 10.129.0.89:7946"
level=debug ts=2026-04-27T23:30:29.719608896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:30:29.72134244Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:30:30.520175868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:30:47.336091547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:34738"
ts=2026-04-27T23:30:47.649575927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:51760"
ts=2026-04-27T23:31:00.52301224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:31:17.33854459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53386"
ts=2026-04-27T23:31:17.65047772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:52634"
level=debug ts=2026-04-27T23:31:17.740164219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:31:30.525181616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=error ts=2026-04-27T23:31:33.346767841Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:31:33.348666024Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:31:33.350242927Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:31:33.350268067Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:31:47.340057552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42728"
ts=2026-04-27T23:32:00.527554299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:32:17.653963297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:54054"
level=debug ts=2026-04-27T23:32:19.03159158Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:32:30.530277988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:32:55.088473081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:33:00.533149526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:33:30.535700097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:33:44.847679082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:33:47.660672144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:48254"
ts=2026-04-27T23:34:00.537619919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55924244 10.129.0.89:7946"
ts=2026-04-27T23:34:09.259393108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:33910"
ts=2026-04-27T23:34:17.662443268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:35210"
level=debug ts=2026-04-27T23:34:19.467914732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:34:19.469822405Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:34:30.539097295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:34:47.664049349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:38068"
ts=2026-04-27T23:35:00.541663943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:35:21.456221149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:35:30.544585991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:36:00.546664682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:36:04.822305926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:36:30.548695446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55924244 10.129.0.89:7946"
level=debug ts=2026-04-27T23:36:52.475431132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:36:59.131784692Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-55924244 (timeout reached)"
ts=2026-04-27T23:37:00.551018365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:37:02.131818734Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-55924244 has failed, no acks received"
ts=2026-04-27T23:37:05.77617434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:58632"
ts=2026-04-27T23:37:05.785014214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:58648"
ts=2026-04-27T23:37:09.130898536Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-55924244 (timeout reached)"
ts=2026-04-27T23:37:11.679248332Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-55924244 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-27T23:37:12.130751806Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-55924244 has failed, no acks received"
ts=2026-04-27T23:37:30.553280589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:37:38.530664297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:37:39.279452521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:43432"
ts=2026-04-27T23:37:47.370608062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39650"
ts=2026-04-27T23:38:00.364829432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:50628"
ts=2026-04-27T23:38:00.555105708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dbab7d5a 10.129.0.89:7946"
level=debug ts=2026-04-27T23:38:20.513750643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:38:20.515370173Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:38:30.365813719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:34468"
ts=2026-04-27T23:38:30.556976393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:38:39.286073979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41538"
ts=2026-04-27T23:39:00.367376816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:51848"
ts=2026-04-27T23:39:00.558186665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dbab7d5a 10.129.0.89:7946"
ts=2026-04-27T23:39:09.288980795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:59620"
level=debug ts=2026-04-27T23:39:20.597645465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:39:30.368312811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:58010"
ts=2026-04-27T23:39:30.559795062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:40:00.561616859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dbab7d5a 10.129.0.89:7946"
level=debug ts=2026-04-27T23:40:08.261202444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:40:09.295267865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53842"
ts=2026-04-27T23:40:30.564018752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:40:43.736544649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:41:00.566597789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-dbab7d5a 10.129.0.89:7946"
ts=2026-04-27T23:41:30.568693234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=error ts=2026-04-27T23:41:33.346652664Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:41:33.348655493Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:41:33.350368649Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:41:33.350394494Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-27T23:41:38.207827011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:42:00.570908272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:42:24.026370291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:42:24.028339547Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:42:30.5727573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:42:39.310253927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42378"
ts=2026-04-27T23:43:00.384709023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:47348"
ts=2026-04-27T23:43:00.575302041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:43:09.312972242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37886"
level=debug ts=2026-04-27T23:43:16.033669703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:43:30.57811001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:43:39.131610453Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-dbab7d5a (timeout reached)"
ts=2026-04-27T23:43:42.131154681Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-dbab7d5a has failed, no acks received"
ts=2026-04-27T23:44:00.580415483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:44:04.131773177Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-dbab7d5a (timeout reached)"
ts=2026-04-27T23:44:07.131138636Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-dbab7d5a has failed, no acks received"
level=debug ts=2026-04-27T23:44:09.44052834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:44:25.752545258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:51536"
ts=2026-04-27T23:44:25.761198461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:51540"
ts=2026-04-27T23:44:30.58251747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:44:47.406009788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:51722"
ts=2026-04-27T23:45:00.584603669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:45:10.065698352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:45:17.408188048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:52812"
ts=2026-04-27T23:45:30.585831831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-11da2ccf 10.129.0.89:7946"
level=debug ts=2026-04-27T23:45:59.331605127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:46:00.588429984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:46:09.332770849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:50092"
ts=2026-04-27T23:46:17.412718147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:40162"
ts=2026-04-27T23:46:30.590625494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:46:47.414863314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:47848"
ts=2026-04-27T23:46:47.845617084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:48382"
level=debug ts=2026-04-27T23:46:52.086086752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:46:52.087899323Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:47:00.592483605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-11da2ccf 10.129.0.89:7946"
ts=2026-04-27T23:47:17.846538182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:51846"
ts=2026-04-27T23:47:30.593692682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-11da2ccf 10.129.0.89:7946"
level=debug ts=2026-04-27T23:47:39.235969903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:47:47.419727032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:42414"
ts=2026-04-27T23:48:00.595730694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:48:17.421896242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:34670"
ts=2026-04-27T23:48:30.59807944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:48:39.06092368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:48:39.347946747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35290"
ts=2026-04-27T23:49:00.600537359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-11da2ccf 10.129.0.89:7946"
ts=2026-04-27T23:49:17.427331945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:44826"
level=debug ts=2026-04-27T23:49:19.933003305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:49:30.60272506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:49:47.429874402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:37378"
ts=2026-04-27T23:50:00.605228347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:50:14.740236726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:50:30.608380931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:50:39.360678454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:47344"
ts=2026-04-27T23:50:47.434602568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:51850"
ts=2026-04-27T23:51:00.610678052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:51:14.320690859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:51:14.322421775Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:51:30.613156784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=error ts=2026-04-27T23:51:33.346380179Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:51:33.348261591Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:51:33.349983369Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-27T23:51:33.35001283Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-27T23:51:52.397989878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:52:00.615301158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-11da2ccf 10.129.0.89:7946"
ts=2026-04-27T23:52:17.441114181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:33564"
ts=2026-04-27T23:52:30.617452024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:52:53.718119714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:53:00.619326491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-11da2ccf 10.129.0.89:7946"
ts=2026-04-27T23:53:24.131882973Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-11da2ccf (timeout reached)"
ts=2026-04-27T23:53:27.131631934Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-11da2ccf has failed, no acks received"
ts=2026-04-27T23:53:30.62074366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:53:39.131218942Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-11da2ccf (timeout reached)"
ts=2026-04-27T23:53:39.378319301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:43536"
ts=2026-04-27T23:53:42.130891338Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-11da2ccf has failed, no acks received"
level=debug ts=2026-04-27T23:53:42.535388194Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:53:47.131804179Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-11da2ccf as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-27T23:53:49.13122855Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-11da2ccf (timeout reached)"
ts=2026-04-27T23:53:52.13103986Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-11da2ccf has failed, no acks received"
ts=2026-04-27T23:54:00.623541093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:54:09.381415859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34106"
level=debug ts=2026-04-27T23:54:17.325645301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:54:28.757595273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:33110"
ts=2026-04-27T23:54:28.766771888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:33112"
ts=2026-04-27T23:54:30.625980519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8def4737 10.129.0.89:7946"
ts=2026-04-27T23:54:47.453422583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:55152"
ts=2026-04-27T23:55:00.62727146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:55:03.477519439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:55:03.479122389Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:55:09.387897786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56152"
ts=2026-04-27T23:55:30.629406891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:55:39.390612659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:35064"
level=debug ts=2026-04-27T23:56:00.303623657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:56:00.631653316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:56:17.460261834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:52296"
ts=2026-04-27T23:56:30.634872744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-27T23:56:44.924241995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:57:00.637873597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:57:09.399278174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:52430"
ts=2026-04-27T23:57:17.465162598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:58698"
level=debug ts=2026-04-27T23:57:26.826244723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:57:30.639895144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:57:39.402861983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:41872"
ts=2026-04-27T23:57:44.180990772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49928"
ts=2026-04-27T23:57:47.467168891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:33772"
ts=2026-04-27T23:58:00.642261357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:58:06.463228929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:58:09.405929471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:44232"
ts=2026-04-27T23:58:30.644344439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-27T23:58:47.471164837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:40182"
level=debug ts=2026-04-27T23:58:49.235705537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-27T23:58:49.237589649Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-27T23:59:00.646579421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-27T23:59:09.411843187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:42058"
ts=2026-04-27T23:59:14.187641501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:40946"
ts=2026-04-27T23:59:30.648841638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-27T23:59:31.429486428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-27T23:59:44.189262692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:52076"
ts=2026-04-28T00:00:00.651006438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8def4737 10.129.0.89:7946"
ts=2026-04-28T00:00:17.478232834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39402"
ts=2026-04-28T00:00:30.65265001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8def4737 10.129.0.89:7946"
level=debug ts=2026-04-28T00:00:33.159540519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:00:44.192146852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:35424"
ts=2026-04-28T00:01:00.654600837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-28T00:01:28.536463043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:01:30.656200771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8def4737 10.129.0.89:7946"
level=error ts=2026-04-28T00:01:33.346502648Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:01:33.348507134Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:01:33.350426136Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:01:33.350451096Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:01:44.13062668Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8def4737 (timeout reached)"
ts=2026-04-28T00:01:47.130643532Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8def4737 has failed, no acks received"
ts=2026-04-28T00:01:47.487104093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:43714"
ts=2026-04-28T00:01:54.131784212Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8def4737 (timeout reached)"
ts=2026-04-28T00:01:57.131225453Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8def4737 has failed, no acks received"
ts=2026-04-28T00:01:57.751989098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:57504"
ts=2026-04-28T00:01:57.761206434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:57512"
ts=2026-04-28T00:02:00.658656858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:02:01.876934076Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-8def4737 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-28T00:02:06.834233948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:02:30.66050909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:02:47.490466291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:33896"
level=debug ts=2026-04-28T00:02:58.198993002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T00:02:58.200694919Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:03:00.662385721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:03:09.436707599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:58106"
ts=2026-04-28T00:03:17.492874782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:44506"
ts=2026-04-28T00:03:30.665584111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:03:39.43961637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34496"
level=debug ts=2026-04-28T00:03:50.782396492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:04:00.668359372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:04:10.228874702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:45602"
ts=2026-04-28T00:04:17.498010217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:39484"
level=debug ts=2026-04-28T00:04:29.52746808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:04:30.670138393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:05:00.672270107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:05:17.502412259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:47716"
level=debug ts=2026-04-28T00:05:24.976545551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:05:30.674810236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:06:00.678529101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:06:10.238465577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:32836"
level=debug ts=2026-04-28T00:06:18.554101125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:06:30.681796362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:06:39.459454412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:51980"
ts=2026-04-28T00:06:40.239738737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:50472"
ts=2026-04-28T00:07:00.684702428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:07:17.511341431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:44066"
level=debug ts=2026-04-28T00:07:18.379621544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T00:07:18.381803761Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:07:30.687568414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:07:40.24385586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49130"
ts=2026-04-28T00:08:00.689205091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-28T00:08:08.378478858Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:08:09.467771801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:56084"
ts=2026-04-28T00:08:10.24470585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:40614"
ts=2026-04-28T00:08:30.690590802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-aed0197e 10.129.0.89:7946"
ts=2026-04-28T00:09:00.692366275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-28T00:09:09.036447237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:09:10.249728151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:48630"
ts=2026-04-28T00:09:30.694750413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:09:40.251331327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:56870"
level=debug ts=2026-04-28T00:09:45.786394986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:10:00.697395308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-aed0197e 10.129.0.89:7946"
ts=2026-04-28T00:10:30.699520073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-28T00:10:32.044539811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:10:34.130894585Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-aed0197e (timeout reached)"
ts=2026-04-28T00:10:37.130924Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-aed0197e has failed, no acks received"
ts=2026-04-28T00:10:39.131501971Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-aed0197e (timeout reached)"
ts=2026-04-28T00:10:42.131071332Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-aed0197e has failed, no acks received"
ts=2026-04-28T00:10:58.760886521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:46594"
ts=2026-04-28T00:10:58.76968143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:46606"
ts=2026-04-28T00:11:00.700605928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f10e3538 10.129.0.89:7946"
level=debug ts=2026-04-28T00:11:12.672055083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T00:11:12.674030421Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:11:30.702853297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=error ts=2026-04-28T00:11:33.34847972Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:11:33.350497145Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:11:33.352368206Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:11:33.352392345Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:11:39.490568104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37616"
ts=2026-04-28T00:12:00.704658947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f10e3538 10.129.0.89:7946"
level=debug ts=2026-04-28T00:12:02.922791078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:12:17.537638655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:53766"
ts=2026-04-28T00:12:30.706123587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:12:39.393636845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49294"
ts=2026-04-28T00:12:39.496582464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:51694"
ts=2026-04-28T00:13:00.70898345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-28T00:13:00.98467174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:13:17.542313217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:37218"
ts=2026-04-28T00:13:30.712049728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-28T00:13:52.208241196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:14:00.714175681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:14:09.401218483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:52076"
ts=2026-04-28T00:14:09.506142718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:32792"
ts=2026-04-28T00:14:30.716751555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f10e3538 10.129.0.89:7946"
level=debug ts=2026-04-28T00:14:34.449268204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:14:39.509462555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34720"
ts=2026-04-28T00:15:00.719170721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:15:09.4052757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:53432"
ts=2026-04-28T00:15:09.512120445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:46226"
level=debug ts=2026-04-28T00:15:18.800851296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T00:15:18.803229842Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:15:30.721158006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f10e3538 10.129.0.89:7946"
ts=2026-04-28T00:15:39.406880738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:50132"
ts=2026-04-28T00:15:39.515729561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:46560"
ts=2026-04-28T00:15:47.553586086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:48158"
level=debug ts=2026-04-28T00:15:59.2144394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:16:00.722940755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:16:30.725927739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-28T00:16:34.458427803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:16:39.52153955Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:57738"
ts=2026-04-28T00:17:00.729417964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-28T00:17:10.667323705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:17:30.731645011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:17:39.415087046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:49936"
ts=2026-04-28T00:18:00.733486297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-f10e3538 10.129.0.89:7946"
level=debug ts=2026-04-28T00:18:08.657882696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:18:30.735488336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:18:39.535428074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:34200"
ts=2026-04-28T00:18:54.131738683Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f10e3538 (timeout reached)"
ts=2026-04-28T00:18:57.13220562Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f10e3538 has failed, no acks received"
ts=2026-04-28T00:19:00.737523469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
level=debug ts=2026-04-28T00:19:05.352561685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T00:19:05.354364152Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:19:09.131515999Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-f10e3538 (timeout reached)"
ts=2026-04-28T00:19:09.538507241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:49876"
ts=2026-04-28T00:19:12.131543042Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-f10e3538 has failed, no acks received"
ts=2026-04-28T00:19:17.133172553Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-f10e3538 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T00:19:29.782774679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:52728"
ts=2026-04-28T00:19:29.792041645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:52742"
ts=2026-04-28T00:19:30.740568467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
level=debug ts=2026-04-28T00:19:58.153111134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:20:00.743518403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:20:30.745348692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55f5db1a 10.129.0.89:7946"
ts=2026-04-28T00:20:34.948536362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:39974"
level=debug ts=2026-04-28T00:20:41.502009006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:20:47.583537195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:35480"
ts=2026-04-28T00:21:00.747800375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:21:17.585689014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:47434"
level=debug ts=2026-04-28T00:21:17.92273534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:21:30.749556868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55f5db1a 10.129.0.89:7946"
level=error ts=2026-04-28T00:21:33.346806723Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:21:33.348411984Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:21:33.35024966Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T00:21:33.350288882Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:21:34.952281567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:45316"
ts=2026-04-28T00:21:47.587701776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:44480"
ts=2026-04-28T00:22:00.750834157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55f5db1a 10.129.0.89:7946"
level=debug ts=2026-04-28T00:22:02.191562957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:22:30.753084299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:22:39.562156962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:37946"
level=debug ts=2026-04-28T00:22:53.865643301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T00:22:53.867108559Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T00:23:00.755240872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:23:17.594235941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:33976"
ts=2026-04-28T00:23:30.756753819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55f5db1a 10.129.0.89:7946"
level=debug ts=2026-04-28T00:23:41.934068263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:24:00.75858444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-wq7hb-315da26f 10.130.0.50:7946"
ts=2026-04-28T00:24:30.760327418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55f5db1a 10.129.0.89:7946"
level=debug ts=2026-04-28T00:24:42.93684852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:25:00.761792054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55f5db1a 10.129.0.89:7946"
ts=2026-04-28T00:25:09.577689056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:53128"
ts=2026-04-28T00:25:17.604675675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:50798"
ts=2026-04-28T00:25:30.76406154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-gdndn-a2714756 10.128.0.88:7946"
ts=2026-04-28T00:25:34.972832845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.89:47148"
ts=2026-04-28T00:25:39.580522271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.88:48006"
level=debug ts=2026-04-28T00:25:42.890788699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T00:26:00.766330803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55f5db1a 10.129.0.89:7946"
ts=2026-04-28T00:26:17.609329684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.50:56282"
ts=2026-04-28T00:26:30.767802269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-55f5db1a 10.129.0.89:7946"
level=debug ts=2026-04-28T00:26:42.272482314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"