level=info ts=2026-04-24T21:32:13.250323601Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-24T21:32:13.250349328Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-24T21:32:13.252286307Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-24T21:32:13.25264359Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-24T21:32:13.252738166Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.71 inf=eth0
level=debug ts=2026-04-24T21:32:13.253161759Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-24T21:32:13.25328197Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.71 inf=eth0
level=info ts=2026-04-24T21:32:13.253506496Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-db3f173d
ts=2026-04-24T21:32:13.253619583Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-24T21:32:13.253638375Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.71 advertisePort=7946
level=debug ts=2026-04-24T21:32:13.253657087Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.71 advertisePort=7946
level=info ts=2026-04-24T21:32:13.25379186Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-24T21:32:13.264428698Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-24T21:32:13.264447075Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-24T21:32:13.264457206Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=10.668136ms
level=error ts=2026-04-24T21:32:13.264465503Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-24T21:32:13.264482514Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-24T21:32:13.26858999Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T21:32:13.268636822Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-24T21:32:13.271721198Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-24T21:32:13.271788401Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-24T21:32:13.271777538Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-24T21:32:13.271798842Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-24T21:32:13.271817752Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-24T21:32:13.271837374Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-24T21:32:13.271857002Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-24T21:32:13.272144161Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-24T21:32:13.272152359Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-24T21:32:13.272155969Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-24T21:32:13.272208319Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-24T21:32:13.272222203Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-24T21:32:13.272506082Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-24T21:32:13.272659684Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-24T21:32:13.272667118Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-24T21:32:13.272680795Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-24T21:32:14.273614536Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-24T21:32:14.452200821Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-24T21:32:14.452267282Z caller=loki.go:599 msg="Loki started" startup_time=1.268261878s
ts=2026-04-24T21:32:14.533344127Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T21:32:14.533369127Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:32:17.548989224Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T21:32:17.549017045Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-24T21:32:19.452908959Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-24T21:32:19.452963448Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-24T21:32:22.604576282Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-24T21:32:22.604599734Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-24T21:32:23.164661493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:32:29.639987802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:58454"
ts=2026-04-24T21:32:31.22043568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49644"
ts=2026-04-24T21:32:32.984284025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40586"
level=debug ts=2026-04-24T21:32:34.688670104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:32:36.297889681Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
ts=2026-04-24T21:32:36.298010445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.71:7946"
ts=2026-04-24T21:32:36.298026698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.71:59216"
level=info ts=2026-04-24T21:32:36.298477084Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=23.033989749s
ts=2026-04-24T21:32:53.05363746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49380"
level=debug ts=2026-04-24T21:32:56.081553796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:32:56.510822951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:33:16.391639024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:33:26.514012558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:33:46.00816709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-24T21:33:46.009803612Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T21:33:53.059466145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42808"
ts=2026-04-24T21:33:56.517828562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:34:11.262461678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:53934"
level=debug ts=2026-04-24T21:34:21.246343068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:34:23.063085748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35376"
ts=2026-04-24T21:34:25.566691764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45040"
ts=2026-04-24T21:34:26.520839566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:34:55.568311986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57212"
ts=2026-04-24T21:34:56.523376143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:35:26.525945055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:35:34.406234393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:35:56.528641426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:36:11.274200638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:56570"
ts=2026-04-24T21:36:25.576821767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49884"
ts=2026-04-24T21:36:26.531977828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T21:36:28.402989117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:36:56.534793779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
level=debug ts=2026-04-24T21:37:16.314108889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-24T21:37:23.078185099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42212"
ts=2026-04-24T21:37:25.582251209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42594"
ts=2026-04-24T21:37:26.536369286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:37:55.583419907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48768"
ts=2026-04-24T21:37:56.537189709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:38:11.285255687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:44370"
ts=2026-04-24T21:38:26.539611729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:38:29.787424263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-24T21:38:29.789375961Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T21:38:56.541279899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
level=debug ts=2026-04-24T21:39:17.16522273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:39:23.08921027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45362"
ts=2026-04-24T21:39:26.542712202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:39:56.545423163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T21:40:05.972710188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:40:11.298701539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:43034"
ts=2026-04-24T21:40:23.095854882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55300"
ts=2026-04-24T21:40:26.549247046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:40:56.552985275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T21:41:07.530571427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:41:11.306753798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45834"
ts=2026-04-24T21:41:25.605052986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43014"
ts=2026-04-24T21:41:26.557062184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:41:41.309881265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35458"
ts=2026-04-24T21:41:53.104174672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38062"
level=debug ts=2026-04-24T21:41:54.180646222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:41:56.560304938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=info ts=2026-04-24T21:42:19.453940429Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-24T21:42:19.459184683Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T21:42:19.461168859Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T21:42:19.462902906Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T21:42:19.462927818Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-24T21:42:19.462938528Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-24T21:42:26.562587967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:42:41.317199278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36550"
level=debug ts=2026-04-24T21:42:48.537856055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T21:42:48.539358363Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T21:42:55.613598802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40162"
ts=2026-04-24T21:42:56.564186191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:43:26.566971684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:43:34.464780468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:43:53.114827916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33420"
ts=2026-04-24T21:43:56.568983079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:44:11.326262129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:59800"
ts=2026-04-24T21:44:25.621946454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38440"
ts=2026-04-24T21:44:26.571059984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:44:32.922643001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:44:56.573121856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
level=debug ts=2026-04-24T21:45:11.106469082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:45:11.333370678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49266"
ts=2026-04-24T21:45:25.627090253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49598"
ts=2026-04-24T21:45:26.574426897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:45:41.337334845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:32938"
ts=2026-04-24T21:45:53.125109322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50000"
ts=2026-04-24T21:45:56.577307196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T21:46:01.499615145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:46:23.128937326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38282"
ts=2026-04-24T21:46:26.57998854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:46:53.131983924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55566"
ts=2026-04-24T21:46:55.635703167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46268"
ts=2026-04-24T21:46:56.582845932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:46:56.786375037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T21:46:56.787997039Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T21:47:26.586388455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:47:43.751157511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:47:56.58946833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:48:25.64447813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40910"
ts=2026-04-24T21:48:26.593491369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T21:48:35.630319031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:48:56.596663059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:49:14.259982806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:49:26.60005586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:49:53.151159093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33220"
ts=2026-04-24T21:49:56.602123125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
level=debug ts=2026-04-24T21:50:12.489978714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:50:26.604254062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:50:41.366349834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:48506"
level=debug ts=2026-04-24T21:50:49.071180495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T21:50:49.073300808Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T21:50:56.607977681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:51:26.611622635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T21:51:49.625739048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:51:56.615154916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:52:11.375353617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33170"
level=error ts=2026-04-24T21:52:19.467562595Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T21:52:19.469621915Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T21:52:19.471515091Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T21:52:19.47156339Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T21:52:23.167775574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:36022"
ts=2026-04-24T21:52:26.618360191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:52:41.379069956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:41404"
level=debug ts=2026-04-24T21:52:51.478611964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:52:53.175608548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39114"
ts=2026-04-24T21:52:55.674239725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51910"
ts=2026-04-24T21:52:56.62038134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
ts=2026-04-24T21:53:25.675259184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45228"
ts=2026-04-24T21:53:26.621847455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
level=debug ts=2026-04-24T21:53:43.233805647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:53:55.676406575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44536"
ts=2026-04-24T21:53:56.624474933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:54:17.933908008Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:54:23.190088729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53888"
ts=2026-04-24T21:54:26.62839326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:54:56.632475184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-04e13d55 10.130.0.69:7946"
level=debug ts=2026-04-24T21:55:00.307371961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T21:55:00.309185752Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T21:55:11.395804324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45466"
ts=2026-04-24T21:55:26.65001993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:55:41.399611451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36748"
level=debug ts=2026-04-24T21:55:50.607040059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:55:53.197893789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:40806"
ts=2026-04-24T21:55:55.691482255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58398"
ts=2026-04-24T21:55:56.65322664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:56:26.656735752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T21:56:28.072794737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:56:41.405154271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:55592"
ts=2026-04-24T21:56:53.202936859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54094"
ts=2026-04-24T21:56:56.660474005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:57:11.408868403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37910"
level=debug ts=2026-04-24T21:57:23.48861073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:57:26.663855132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:57:44.98523871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47220"
ts=2026-04-24T21:57:44.99727707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47234"
ts=2026-04-24T21:57:50.254046107Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-04e13d55 (timeout reached)"
ts=2026-04-24T21:57:53.254614216Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-04e13d55 has failed, no acks received"
ts=2026-04-24T21:57:56.66705447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T21:58:00.598699045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:58:26.101884175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:54228"
ts=2026-04-24T21:58:26.670695872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T21:58:56.674892187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T21:59:02.567924555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T21:59:02.569899179Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T21:59:23.218683349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41926"
ts=2026-04-24T21:59:26.107840035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59760"
ts=2026-04-24T21:59:26.678708377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T21:59:41.428808428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:45212"
level=debug ts=2026-04-24T21:59:43.852404574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T21:59:53.222174103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55394"
ts=2026-04-24T21:59:56.680613132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cef27677 10.130.0.69:7946"
ts=2026-04-24T22:00:26.682696874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:00:30.857258382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:00:53.228267187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45762"
ts=2026-04-24T22:00:56.685152437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cef27677 10.130.0.69:7946"
ts=2026-04-24T22:01:26.687278134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:01:32.302736922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:01:56.690597194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:02:11.338646396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:02:19.466236923Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:02:19.468138599Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:02:19.469947214Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:02:19.469972167Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:02:23.237418546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35750"
ts=2026-04-24T22:02:26.126638469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49790"
ts=2026-04-24T22:02:26.693509083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:02:53.240770754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41556"
ts=2026-04-24T22:02:56.12782106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45756"
ts=2026-04-24T22:02:56.696562817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:03:04.822816439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:03:04.824524316Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:03:23.243734022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:50884"
ts=2026-04-24T22:03:26.128850854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59952"
ts=2026-04-24T22:03:26.69988945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:03:41.454186872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:52388"
level=debug ts=2026-04-24T22:03:49.517695705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:03:56.703408563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:04:26.706014951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cef27677 10.130.0.69:7946"
level=debug ts=2026-04-24T22:04:41.399514257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:04:56.70890039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:05:26.147918954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46002"
ts=2026-04-24T22:05:26.711209886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cef27677 10.130.0.69:7946"
level=debug ts=2026-04-24T22:05:35.505227578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:05:56.712786658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cef27677 10.130.0.69:7946"
level=debug ts=2026-04-24T22:06:17.940441942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:06:26.152361228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57358"
ts=2026-04-24T22:06:26.714714753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cef27677 10.130.0.69:7946"
ts=2026-04-24T22:06:56.717025197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:06:56.886883445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:06:56.888850963Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:07:11.47884009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:59144"
ts=2026-04-24T22:07:26.157308502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44934"
ts=2026-04-24T22:07:26.719058131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-cef27677 10.130.0.69:7946"
ts=2026-04-24T22:07:56.721895991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:07:58.423524566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:08:23.275751116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45886"
ts=2026-04-24T22:08:26.725044443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:08:35.361219998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:08:37.241149881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44072"
ts=2026-04-24T22:08:37.254941643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44088"
ts=2026-04-24T22:08:45.254889577Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cef27677 (timeout reached)"
ts=2026-04-24T22:08:48.255526468Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cef27677 has failed, no acks received"
ts=2026-04-24T22:08:53.278675135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45616"
ts=2026-04-24T22:08:55.25521903Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-cef27677 (timeout reached)"
ts=2026-04-24T22:08:56.729060721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:08:57.878619652Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-cef27677 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T22:08:58.25520769Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-cef27677 has failed, no acks received"
level=debug ts=2026-04-24T22:09:12.676567747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:09:26.732977122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:09:40.338884255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53624"
ts=2026-04-24T22:09:53.284469694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37694"
ts=2026-04-24T22:09:56.735723323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-16076b27 10.130.0.69:7946"
level=debug ts=2026-04-24T22:10:14.979580806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:10:23.287895472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56810"
ts=2026-04-24T22:10:26.738036409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:10:53.292370617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41682"
ts=2026-04-24T22:10:56.741664273Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:11:02.021198316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:11:02.022877903Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:11:23.296308821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48294"
ts=2026-04-24T22:11:26.74420119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-16076b27 10.130.0.69:7946"
ts=2026-04-24T22:11:40.352975177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50384"
ts=2026-04-24T22:11:41.507507896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:51390"
level=debug ts=2026-04-24T22:11:49.75126047Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:11:56.745764154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-16076b27 10.130.0.69:7946"
ts=2026-04-24T22:12:10.354010194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46742"
level=error ts=2026-04-24T22:12:19.466421354Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:12:19.468497889Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:12:19.47031259Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:12:19.470339735Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:12:23.301068403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:38834"
level=debug ts=2026-04-24T22:12:24.963749709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:12:26.748916151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:12:56.752664413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:13:15.107419751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:13:26.756364713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:13:40.361259723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43248"
ts=2026-04-24T22:13:41.518799462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:35098"
ts=2026-04-24T22:13:56.76057339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:14:01.135446485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:14:23.31282023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48468"
ts=2026-04-24T22:14:26.763980285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:14:40.366069247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55112"
level=debug ts=2026-04-24T22:14:45.392409439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:14:45.394025929Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:14:56.767816141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:15:23.319097308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45588"
ts=2026-04-24T22:15:26.771580138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:15:41.531720525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:51442"
level=debug ts=2026-04-24T22:15:45.195258625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:15:56.774656313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:16:26.780655996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:16:35.176468899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:16:56.783036545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-16076b27 10.130.0.69:7946"
ts=2026-04-24T22:17:10.38548101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35364"
level=debug ts=2026-04-24T22:17:20.01127102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:17:26.78577304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:17:40.386760496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41800"
ts=2026-04-24T22:17:41.545300098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33444"
ts=2026-04-24T22:17:55.254459781Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-16076b27 (timeout reached)"
ts=2026-04-24T22:17:56.788349609Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-16076b27 failed: dial tcp 10.130.0.69:7946: connect: connection refused"
ts=2026-04-24T22:17:58.254605512Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-16076b27 has failed, no acks received"
ts=2026-04-24T22:18:00.651876748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59272"
ts=2026-04-24T22:18:00.664607065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59274"
level=debug ts=2026-04-24T22:18:05.749703297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:18:11.548608879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:56636"
ts=2026-04-24T22:18:15.254830708Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-16076b27 (timeout reached)"
ts=2026-04-24T22:18:18.254655857Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-16076b27 has failed, no acks received"
ts=2026-04-24T22:18:23.335862733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52984"
ts=2026-04-24T22:18:26.790827915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:18:53.338795006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60242"
level=debug ts=2026-04-24T22:18:55.750322941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:18:55.752018616Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:18:56.793994202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:19:13.089444845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34126"
ts=2026-04-24T22:19:26.797156734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:19:43.090923573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49392"
level=debug ts=2026-04-24T22:19:51.2063949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:19:56.799176184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51af2127 10.130.0.69:7946"
ts=2026-04-24T22:20:11.561887014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:48596"
ts=2026-04-24T22:20:26.801889731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:20:27.543394407Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:20:41.565396626Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:33598"
ts=2026-04-24T22:20:53.351221966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57250"
ts=2026-04-24T22:20:56.805219579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:21:11.568666313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:34286"
level=debug ts=2026-04-24T22:21:16.816938523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:21:23.354769976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49148"
ts=2026-04-24T22:21:26.809481361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:21:55.52747902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:21:56.813108935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:22:11.576032397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36044"
ts=2026-04-24T22:22:13.105676617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50124"
level=error ts=2026-04-24T22:22:19.466894323Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:22:19.468812077Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:22:19.470401461Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:22:19.470430704Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:22:26.815135646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51af2127 10.130.0.69:7946"
ts=2026-04-24T22:22:41.579953377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:57810"
level=debug ts=2026-04-24T22:22:50.050715672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:22:50.052653551Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:22:56.817260791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:23:23.366862185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59308"
ts=2026-04-24T22:23:26.819499098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-51af2127 10.130.0.69:7946"
level=debug ts=2026-04-24T22:23:30.135838389Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:23:56.822477794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:24:23.370879032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:24:26.82633657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:24:41.593110186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49542"
ts=2026-04-24T22:24:50.254660881Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-51af2127 (timeout reached)"
ts=2026-04-24T22:24:53.254773106Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-51af2127 has failed, no acks received"
ts=2026-04-24T22:24:53.374506456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48836"
ts=2026-04-24T22:24:56.830254433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:25:00.254709619Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-51af2127 (timeout reached)"
ts=2026-04-24T22:25:03.254753557Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-51af2127 has failed, no acks received"
ts=2026-04-24T22:25:10.255660241Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-51af2127 (timeout reached)"
level=debug ts=2026-04-24T22:25:11.462920039Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:25:11.596315636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49624"
ts=2026-04-24T22:25:11.638303567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34334"
ts=2026-04-24T22:25:11.65025114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34336"
ts=2026-04-24T22:25:13.254933545Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-51af2127 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T22:25:13.25498673Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-51af2127 has failed, no acks received"
ts=2026-04-24T22:25:26.833888853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:25:56.835860045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00fe1811 10.130.0.69:7946"
level=debug ts=2026-04-24T22:26:05.816893116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:26:26.837839124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:26:52.858439519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:26:52.860386312Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:26:53.383966527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45734"
ts=2026-04-24T22:26:56.841550095Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:27:22.528077869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51694"
ts=2026-04-24T22:27:26.843806696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00fe1811 10.130.0.69:7946"
level=debug ts=2026-04-24T22:27:33.585277998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:27:52.529856205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43228"
ts=2026-04-24T22:27:56.846714554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:28:26.850729114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:28:28.793058463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:28:56.852937802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00fe1811 10.130.0.69:7946"
level=debug ts=2026-04-24T22:29:06.800308079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:29:11.619520299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:56748"
ts=2026-04-24T22:29:23.397189524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:39230"
ts=2026-04-24T22:29:26.855638626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:29:41.623425977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:50410"
level=debug ts=2026-04-24T22:29:52.026557556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:29:56.858256939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00fe1811 10.130.0.69:7946"
ts=2026-04-24T22:30:11.627423117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:34504"
ts=2026-04-24T22:30:22.545143632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35898"
ts=2026-04-24T22:30:26.860887433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:30:31.858697443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:30:31.860331872Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:30:41.631425756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:57006"
ts=2026-04-24T22:30:56.863685537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00fe1811 10.130.0.69:7946"
level=debug ts=2026-04-24T22:31:11.086856292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:31:11.634910737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:52248"
ts=2026-04-24T22:31:26.86541597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00fe1811 10.130.0.69:7946"
ts=2026-04-24T22:31:41.63785515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:58434"
level=debug ts=2026-04-24T22:31:47.483863447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:31:53.415744809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:60266"
ts=2026-04-24T22:31:56.866763895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-00fe1811 10.130.0.69:7946"
level=error ts=2026-04-24T22:32:19.467197817Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:32:19.46919354Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:32:19.471120191Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:32:19.47115085Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:32:23.418764558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53566"
ts=2026-04-24T22:32:26.869128503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:32:35.835037544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:32:53.42239432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42866"
ts=2026-04-24T22:32:56.87312552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:33:13.009468863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:33:22.564604433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45180"
ts=2026-04-24T22:33:23.425688162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:48630"
ts=2026-04-24T22:33:26.876845845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:33:55.254819364Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-00fe1811 (timeout reached)"
ts=2026-04-24T22:33:56.879746414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:33:58.255199067Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-00fe1811 has failed, no acks received"
level=debug ts=2026-04-24T22:34:03.155401507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:34:03.156959608Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:34:05.254814048Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-00fe1811 (timeout reached)"
ts=2026-04-24T22:34:08.255325802Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-00fe1811 has failed, no acks received"
ts=2026-04-24T22:34:15.255714154Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-00fe1811 (timeout reached)"
ts=2026-04-24T22:34:17.878783952Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-00fe1811 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T22:34:18.25568048Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-00fe1811 has failed, no acks received"
ts=2026-04-24T22:34:26.88325534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:34:37.634173267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60832"
ts=2026-04-24T22:34:37.647400778Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60834"
ts=2026-04-24T22:34:53.433136607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:37196"
level=debug ts=2026-04-24T22:34:55.08394524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:34:56.885500519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-694b48f5 10.130.0.69:7946"
ts=2026-04-24T22:35:23.43610542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:51524"
ts=2026-04-24T22:35:26.886653551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-694b48f5 10.130.0.69:7946"
level=debug ts=2026-04-24T22:35:40.099499632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:35:53.439253426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59232"
ts=2026-04-24T22:35:56.88842027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-694b48f5 10.130.0.69:7946"
ts=2026-04-24T22:36:26.89113348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:36:27.937676625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:36:53.444559842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:49596"
ts=2026-04-24T22:36:56.89451689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:37:19.705856055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47996"
level=debug ts=2026-04-24T22:37:21.482803705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:37:26.898024729Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:37:49.707703989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40612"
ts=2026-04-24T22:37:56.901135303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:38:07.663812136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:38:07.665452318Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:38:26.904005532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:38:49.712087265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38558"
ts=2026-04-24T22:38:53.455150298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:59916"
ts=2026-04-24T22:38:56.90729376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:39:01.149384809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:39:26.910753701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:39:41.68380198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:44944"
ts=2026-04-24T22:39:49.717087493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47396"
level=debug ts=2026-04-24T22:39:56.135665537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:39:56.91389303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:40:19.718851718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43352"
ts=2026-04-24T22:40:26.916443703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-694b48f5 10.130.0.69:7946"
level=debug ts=2026-04-24T22:40:33.534820742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:40:49.719956044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43740"
ts=2026-04-24T22:40:56.918441439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:41:16.848151615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:41:19.72102638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52020"
ts=2026-04-24T22:41:26.921495143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:41:53.473074675Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54914"
ts=2026-04-24T22:41:56.92519672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:42:07.275519445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:42:07.277126902Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:42:19.467251218Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:42:19.469220288Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:42:19.471146623Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:42:19.471170234Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:42:26.928618806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:42:49.728959304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40340"
ts=2026-04-24T22:42:53.478100953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:47908"
level=debug ts=2026-04-24T22:42:56.171397437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:42:56.931343579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:43:19.730070004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33682"
ts=2026-04-24T22:43:26.93424664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:43:42.5584291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:43:49.731728652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:37958"
ts=2026-04-24T22:43:56.93784105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:44:11.71217711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49332"
ts=2026-04-24T22:44:26.940836305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:44:41.715637702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:43094"
level=debug ts=2026-04-24T22:44:41.980326747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:44:50.168374245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44376"
ts=2026-04-24T22:44:50.179165897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44386"
ts=2026-04-24T22:44:53.490318236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:35278"
ts=2026-04-24T22:44:56.943195701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-694b48f5 10.130.0.69:7946"
ts=2026-04-24T22:45:10.254736299Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-694b48f5 (timeout reached)"
ts=2026-04-24T22:45:13.254272562Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-694b48f5 has failed, no acks received"
ts=2026-04-24T22:45:23.493936142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:41374"
ts=2026-04-24T22:45:26.94523504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:45:27.928159451Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-694b48f5 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-24T22:45:33.124730704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:45:41.72126176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:36176"
ts=2026-04-24T22:45:56.947629086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
level=debug ts=2026-04-24T22:46:25.124951601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:46:25.126825972Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:46:26.948829141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
ts=2026-04-24T22:46:31.055641453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51996"
ts=2026-04-24T22:46:53.501571178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57704"
ts=2026-04-24T22:46:56.951287654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:46:59.955189037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:47:01.056763131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53684"
ts=2026-04-24T22:47:11.730904168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:50724"
ts=2026-04-24T22:47:26.954113896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:47:37.391172449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:47:41.73488082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:47914"
ts=2026-04-24T22:47:56.957365455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:48:22.470765677Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:48:26.959787675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
ts=2026-04-24T22:48:41.741100051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:51594"
ts=2026-04-24T22:48:56.9626302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:49:18.22317124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:49:23.514793427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53464"
ts=2026-04-24T22:49:26.965613621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:49:56.969329867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:50:11.750796198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:32982"
level=debug ts=2026-04-24T22:50:19.087573691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:50:19.089235478Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:50:23.521840179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:56620"
ts=2026-04-24T22:50:26.972451584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:50:41.754369268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:49080"
ts=2026-04-24T22:50:56.975035714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
ts=2026-04-24T22:51:01.082749801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52418"
level=debug ts=2026-04-24T22:51:05.02968175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:51:26.976708926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
level=debug ts=2026-04-24T22:51:49.472263081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:51:53.531143806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55558"
ts=2026-04-24T22:51:56.978026524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
ts=2026-04-24T22:52:01.08744709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57552"
level=error ts=2026-04-24T22:52:19.466556529Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:52:19.468571738Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:52:19.470839991Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T22:52:19.470870617Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:52:26.98079362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:52:47.719878745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:52:56.983970285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:53:25.102711415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:53:26.986379443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
ts=2026-04-24T22:53:41.76760854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:53590"
ts=2026-04-24T22:53:53.544001637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:54026"
ts=2026-04-24T22:53:56.987176614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
level=debug ts=2026-04-24T22:54:11.136847726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:54:11.138400583Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:54:26.988327292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-98a7c36a 10.130.0.69:7946"
ts=2026-04-24T22:54:53.549939455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:52240"
ts=2026-04-24T22:54:56.99098218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:55:06.23179565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:55:14.75633629Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59264"
ts=2026-04-24T22:55:14.768575578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59268"
ts=2026-04-24T22:55:26.994362388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T22:55:40.255203119Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-98a7c36a (timeout reached)"
ts=2026-04-24T22:55:43.254840254Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-98a7c36a has failed, no acks received"
level=debug ts=2026-04-24T22:55:45.790174507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:55:53.555551865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53772"
ts=2026-04-24T22:55:56.996766668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8a9e604e 10.130.0.69:7946"
ts=2026-04-24T22:55:57.791422015Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59444"
ts=2026-04-24T22:56:26.998261433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8a9e604e 10.130.0.69:7946"
level=debug ts=2026-04-24T22:56:35.986662693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:56:53.560761807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34182"
ts=2026-04-24T22:56:57.000614101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:57:26.484852248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:57:27.002567432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8a9e604e 10.130.0.69:7946"
ts=2026-04-24T22:57:41.791996773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:53704"
ts=2026-04-24T22:57:57.005546453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T22:58:05.320154547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T22:58:05.321687973Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T22:58:11.79549633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:51744"
ts=2026-04-24T22:58:23.569159369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58442"
ts=2026-04-24T22:58:27.00880717Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T22:58:52.821229182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:58:57.012169519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T22:59:11.801768412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:46228"
ts=2026-04-24T22:59:27.014794132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8a9e604e 10.130.0.69:7946"
level=debug ts=2026-04-24T22:59:35.890988491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T22:59:53.578441791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:42624"
ts=2026-04-24T22:59:57.015830827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8a9e604e 10.130.0.69:7946"
level=debug ts=2026-04-24T23:00:17.185146767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:00:27.018625912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T23:00:53.584972814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:34992"
ts=2026-04-24T23:00:57.0217476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T23:01:18.205068877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:01:27.02502028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T23:01:41.816060233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:60094"
level=debug ts=2026-04-24T23:01:53.84657956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:01:53.848240982Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:01:55.254662815Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8a9e604e (timeout reached)"
ts=2026-04-24T23:01:57.028771452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T23:01:58.255403559Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8a9e604e has failed, no acks received"
ts=2026-04-24T23:01:59.641364407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39466"
ts=2026-04-24T23:01:59.654506849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39470"
ts=2026-04-24T23:02:05.255682175Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8a9e604e (timeout reached)"
ts=2026-04-24T23:02:08.255810363Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8a9e604e has failed, no acks received"
ts=2026-04-24T23:02:08.929098616Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-8a9e604e as failed, suspect timeout reached (2 peer confirmations)"
level=error ts=2026-04-24T23:02:19.466670151Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:02:19.468551398Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:02:19.470711829Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-24T23:02:19.47073551Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:02:27.031957975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T23:02:28.406139134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:02:33.88776672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47178"
ts=2026-04-24T23:02:41.821660729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:52644"
ts=2026-04-24T23:02:57.035196237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
level=debug ts=2026-04-24T23:03:21.819090593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:03:27.03931984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T23:03:33.892809769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35646"
ts=2026-04-24T23:03:41.82701044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:41946"
ts=2026-04-24T23:03:57.043157738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T23:04:03.894203642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33100"
level=debug ts=2026-04-24T23:04:12.79732041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:04:23.603031292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:58964"
ts=2026-04-24T23:04:27.046697864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T23:04:33.895759549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41314"
ts=2026-04-24T23:04:57.050214387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T23:04:58.568491425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:05:27.05277177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-79217b68 10.130.0.69:7946"
ts=2026-04-24T23:05:57.0541245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-79217b68 10.130.0.69:7946"
level=debug ts=2026-04-24T23:05:59.049481657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:05:59.051151747Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:06:23.613970808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:44990"
ts=2026-04-24T23:06:27.055574633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-79217b68 10.130.0.69:7946"
ts=2026-04-24T23:06:33.908080726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42218"
ts=2026-04-24T23:06:41.844714389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:48332"
ts=2026-04-24T23:06:53.616784612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:46940"
level=debug ts=2026-04-24T23:06:56.055844906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:06:57.057822523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T23:07:03.909107879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43934"
ts=2026-04-24T23:07:11.848558331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:59072"
ts=2026-04-24T23:07:27.059873471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-79217b68 10.130.0.69:7946"
ts=2026-04-24T23:07:41.85234094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:57054"
ts=2026-04-24T23:07:53.621513237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:53516"
level=debug ts=2026-04-24T23:07:55.147187086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:07:57.061708767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-79217b68 10.130.0.69:7946"
ts=2026-04-24T23:08:03.914124922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45760"
ts=2026-04-24T23:08:11.85630891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:37970"
ts=2026-04-24T23:08:27.063847641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T23:08:41.859625776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:54780"
level=debug ts=2026-04-24T23:08:45.542863238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:08:57.066435264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-79217b68 10.130.0.69:7946"
ts=2026-04-24T23:09:03.91883255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42902"
ts=2026-04-24T23:09:23.629479793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:57970"
ts=2026-04-24T23:09:27.068740506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
level=debug ts=2026-04-24T23:09:29.729820754Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:09:41.865655825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.75:43958"
ts=2026-04-24T23:09:53.633175478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:45676"
ts=2026-04-24T23:09:57.071247267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-79217b68 10.130.0.69:7946"
level=debug ts=2026-04-24T23:10:22.157239696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-24T23:10:22.158855943Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-24T23:10:27.07352114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-czw56-0e632376 10.129.0.67:7946"
ts=2026-04-24T23:10:57.07567754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-79217b68 10.130.0.69:7946"
level=debug ts=2026-04-24T23:11:04.71696929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-24T23:11:10.255813383Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-79217b68 (timeout reached)"
ts=2026-04-24T23:11:13.255785652Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-79217b68 has failed, no acks received"
ts=2026-04-24T23:11:20.255216443Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-79217b68 (timeout reached)"
ts=2026-04-24T23:11:23.255552459Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-79217b68 has failed, no acks received"
ts=2026-04-24T23:11:23.641275595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:33268"
ts=2026-04-24T23:11:27.078172158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-rt55x-f0422f23 10.128.0.75:7946"
ts=2026-04-24T23:11:33.255918505Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-79217b68 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-24T23:11:35.637303665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39804"
ts=2026-04-24T23:11:35.650220335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39812"