level=info ts=2026-04-21T17:40:17.292196536Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T17:40:17.292227034Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-21T17:40:17.293116862Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T17:40:17.293455155Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T17:40:17.293541557Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.67 inf=eth0
level=debug ts=2026-04-21T17:40:17.294506518Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T17:40:17.294584064Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.67 inf=eth0
level=info ts=2026-04-21T17:40:17.294663129Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-aeb73ee7
ts=2026-04-21T17:40:17.294748565Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T17:40:17.294761318Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.67 advertisePort=7946
level=debug ts=2026-04-21T17:40:17.294781121Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.67 advertisePort=7946
level=info ts=2026-04-21T17:40:17.294883111Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-21T17:40:17.300622923Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T17:40:17.300647527Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T17:40:17.300658207Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.77756ms
level=error ts=2026-04-21T17:40:17.300666043Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T17:40:17.300678723Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T17:40:17.302847122Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T17:40:17.302864095Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T17:40:17.309035944Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-21T17:40:17.309034373Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-21T17:40:17.309045125Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-21T17:40:17.30906137Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-21T17:40:17.309060389Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-21T17:40:17.309085095Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T17:40:17.309180266Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-21T17:40:17.309190753Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-21T17:40:17.309334889Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-21T17:40:17.309352957Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-21T17:40:17.309402816Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-21T17:40:17.309437583Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-21T17:40:17.309452776Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T17:40:17.309564973Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-21T17:40:17.309572729Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-21T17:40:17.309586594Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-21T17:40:18.31042363Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-21T17:40:18.510044634Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-21T17:40:18.510101335Z caller=loki.go:599 msg="Loki started" startup_time=1.273545371s
ts=2026-04-21T17:40:19.132544116Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T17:40:19.132570395Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:40:21.489032614Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T17:40:21.489059781Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T17:40:23.510511089Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-21T17:40:23.510570728Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-21T17:40:25.913651816Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T17:40:25.913677601Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T17:40:33.009417409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:40:39.491761936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.67:7946"
ts=2026-04-21T17:40:39.491866538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.67:55800"
ts=2026-04-21T17:40:39.494183307Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
ts=2026-04-21T17:40:39.49573271Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.71:7946: dial tcp 10.128.0.71:7946: connect: connection refused"
level=info ts=2026-04-21T17:40:39.495744256Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=22.195059654s
ts=2026-04-21T17:40:45.694909454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:44972"
ts=2026-04-21T17:40:47.326000794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:42332"
ts=2026-04-21T17:40:48.746727523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:47008"
level=debug ts=2026-04-21T17:40:52.589822188Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:40:55.938217672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T17:41:07.200215637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:51438"
level=debug ts=2026-04-21T17:41:12.418305721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:41:25.940553535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T17:41:41.46111313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:41:55.94271073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T17:42:02.707516322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:48384"
ts=2026-04-21T17:42:07.203528242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:44050"
level=debug ts=2026-04-21T17:42:13.188221308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T17:42:13.189968864Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:42:25.944704702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T17:42:45.09238948Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:42:54.211609878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:41342"
ts=2026-04-21T17:42:55.947313508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T17:43:02.711574752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44746"
ts=2026-04-21T17:43:25.949807203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:43:37.208236684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:39396"
ts=2026-04-21T17:43:55.952262544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T17:43:59.020110857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:44:07.210522305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:36180"
ts=2026-04-21T17:44:25.95485787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T17:44:37.213237241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:43396"
ts=2026-04-21T17:44:54.216546243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:55476"
level=debug ts=2026-04-21T17:44:54.378380615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:44:55.957525876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T17:45:02.718285737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:57320"
ts=2026-04-21T17:45:07.21461314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:33796"
ts=2026-04-21T17:45:25.95986007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T17:45:37.216877948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:49226"
level=debug ts=2026-04-21T17:45:48.120470302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T17:45:54.219520313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:37630"
ts=2026-04-21T17:45:55.961827714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:46:07.219262266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:56306"
ts=2026-04-21T17:46:24.221374599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:60878"
ts=2026-04-21T17:46:25.964263774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:46:32.723412965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:47024"
ts=2026-04-21T17:46:54.22417081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:39492"
ts=2026-04-21T17:46:55.966109163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=debug ts=2026-04-21T17:46:59.057712348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T17:46:59.059496523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:47:25.968186727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T17:47:32.726998179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:45822"
level=debug ts=2026-04-21T17:47:36.521319369Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:47:55.970072268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:48:07.226290644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:38256"
ts=2026-04-21T17:48:25.972094845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=debug ts=2026-04-21T17:48:36.216230339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:48:55.974700117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T17:49:02.73236174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52548"
ts=2026-04-21T17:49:24.232514083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:51380"
ts=2026-04-21T17:49:25.976965874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T17:49:29.471002492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:49:32.73467114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:56658"
ts=2026-04-21T17:49:37.232184344Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:41382"
ts=2026-04-21T17:49:55.979548238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=info ts=2026-04-21T17:50:23.510877586Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-21T17:50:23.517672434Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:50:23.519847499Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:50:23.522195376Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T17:50:23.522257277Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T17:50:23.522274152Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-21T17:50:24.236267201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:35296"
ts=2026-04-21T17:50:25.982211041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T17:50:27.951853205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:50:55.984279295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T17:51:02.73916291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39810"
level=debug ts=2026-04-21T17:51:23.427317437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:51:23.4295764Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:51:25.987043525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:51:55.988843545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T17:52:00.804198151Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:52:25.990407155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:52:37.242188122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:59430"
level=debug ts=2026-04-21T17:52:47.096793617Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:52:55.992770604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:53:02.746323565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:41536"
ts=2026-04-21T17:53:25.995510939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T17:53:29.377607372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:53:37.245358739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:56764"
ts=2026-04-21T17:53:54.251125856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:37678"
ts=2026-04-21T17:53:55.997224281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T17:54:07.24699678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:49562"
level=debug ts=2026-04-21T17:54:13.625221401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:54:24.253341833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:56558"
ts=2026-04-21T17:54:26.000031632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T17:54:54.255431863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:54016"
ts=2026-04-21T17:54:56.00192335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=debug ts=2026-04-21T17:55:01.347578727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:55:01.350362287Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:55:26.003770934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T17:55:56.005894642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=debug ts=2026-04-21T17:55:59.516839833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:56:02.756397057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35086"
ts=2026-04-21T17:56:26.00755661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:56:37.255087505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:58714"
ts=2026-04-21T17:56:56.009953166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T17:56:57.938520869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:57:02.759797422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:59520"
ts=2026-04-21T17:57:07.256989619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:42394"
ts=2026-04-21T17:57:26.012600268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=debug ts=2026-04-21T17:57:47.78886653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:57:54.265328786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:55452"
ts=2026-04-21T17:57:56.014422266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:58:26.016523357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T17:58:37.262336638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:32910"
level=debug ts=2026-04-21T17:58:48.545261088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T17:58:56.018446467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T17:59:23.227278183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T17:59:23.228840076Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T17:59:24.270207366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:45292"
ts=2026-04-21T17:59:26.020585616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T17:59:32.767216657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:48502"
ts=2026-04-21T17:59:56.02269156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:00:17.144252425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:00:23.526158615Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:00:23.527944973Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:00:23.529607375Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:00:23.529631384Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:00:24.274143742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:56346"
ts=2026-04-21T18:00:26.025064511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T18:00:54.276193047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:47794"
ts=2026-04-21T18:00:56.027720282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=debug ts=2026-04-21T18:01:00.702043979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:01:24.278452956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:55184"
ts=2026-04-21T18:01:26.029981695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T18:01:32.77408142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52964"
level=debug ts=2026-04-21T18:01:41.387102356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:01:54.285795722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:41566"
ts=2026-04-21T18:01:56.032449416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T18:02:02.776257299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:59696"
level=debug ts=2026-04-21T18:02:22.9928714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:02:26.036409158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:02:32.778716129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:41398"
ts=2026-04-21T18:02:56.052058931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:03:02.78136971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:56112"
level=debug ts=2026-04-21T18:03:16.3390503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:03:16.341389134Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:03:26.054133175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:03:32.787472074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60998"
ts=2026-04-21T18:03:37.281594962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:54418"
ts=2026-04-21T18:03:54.294341319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:35430"
ts=2026-04-21T18:03:56.056738923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=debug ts=2026-04-21T18:04:05.457496156Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:04:24.296571212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:47512"
ts=2026-04-21T18:04:26.0591684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:04:32.791114196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:50540"
level=debug ts=2026-04-21T18:04:41.972540071Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:04:56.061749516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:05:02.793474959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52508"
ts=2026-04-21T18:05:24.300305959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:44048"
ts=2026-04-21T18:05:26.064253931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:05:32.59355949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:05:56.066601007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:06:26.068637282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:06:26.650971355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:06:37.293352567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:48292"
ts=2026-04-21T18:06:54.305277346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:58924"
ts=2026-04-21T18:06:56.070497243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:07:07.295323444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:32934"
level=debug ts=2026-04-21T18:07:07.60292646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:07:07.604452417Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:07:26.07208844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:07:32.801508057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:34116"
ts=2026-04-21T18:07:37.297690253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50546"
ts=2026-04-21T18:07:56.074482375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T18:08:02.80383789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39856"
ts=2026-04-21T18:08:07.300074067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50814"
level=debug ts=2026-04-21T18:08:08.165683997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:08:26.076730039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
ts=2026-04-21T18:08:37.301597152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:55594"
ts=2026-04-21T18:08:56.078545283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-216abae7 10.130.0.93:7946"
level=debug ts=2026-04-21T18:08:57.63089708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:09:26.080463186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:09:37.304254397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:39428"
ts=2026-04-21T18:09:50.591803558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:45664"
ts=2026-04-21T18:09:50.607378663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:45666"
ts=2026-04-21T18:09:56.083825936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
level=debug ts=2026-04-21T18:09:57.422405258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:10:14.295989491Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-216abae7 (timeout reached)"
ts=2026-04-21T18:10:17.296093368Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-216abae7 has failed, no acks received"
level=error ts=2026-04-21T18:10:23.527506979Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:10:23.529461355Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:10:23.531249926Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:10:23.531285361Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:10:26.085476615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:10:38.329053347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:10:56.0874976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
level=debug ts=2026-04-21T18:11:22.12771562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:11:22.129695761Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:11:24.320209446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:52936"
ts=2026-04-21T18:11:26.090211414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:11:34.268808442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:47666"
ts=2026-04-21T18:11:56.093673531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
ts=2026-04-21T18:12:02.818456581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:59416"
ts=2026-04-21T18:12:04.271354791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:44672"
level=debug ts=2026-04-21T18:12:20.504731442Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:12:26.096491139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
ts=2026-04-21T18:12:54.324971961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:49248"
ts=2026-04-21T18:12:56.098853843Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:12:59.377477987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:13:24.327357586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:60682"
ts=2026-04-21T18:13:26.100331539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
ts=2026-04-21T18:13:34.275967176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:41218"
level=debug ts=2026-04-21T18:13:50.500337575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:13:56.102519587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
ts=2026-04-21T18:14:02.826380328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:50320"
level=debug ts=2026-04-21T18:14:25.431780473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:14:26.104542636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
ts=2026-04-21T18:14:32.828542975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:37810"
ts=2026-04-21T18:14:34.279315387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:41202"
ts=2026-04-21T18:14:56.107113549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:15:18.734356962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:15:18.736640956Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:15:26.1093513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
ts=2026-04-21T18:15:34.285980009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:53970"
ts=2026-04-21T18:15:56.111619323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:16:15.2129524Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:16:26.113340769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-081cb414 10.130.0.93:7946"
ts=2026-04-21T18:16:34.290095971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:58510"
ts=2026-04-21T18:16:54.295348215Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-081cb414 (timeout reached)"
ts=2026-04-21T18:16:56.115524169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:16:56.632683792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:16:57.295683724Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-081cb414 has failed, no acks received"
ts=2026-04-21T18:17:08.821519995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50392"
ts=2026-04-21T18:17:08.832385349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50400"
ts=2026-04-21T18:17:09.296729305Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-081cb414 (timeout reached)"
ts=2026-04-21T18:17:12.296003287Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-081cb414 has failed, no acks received"
ts=2026-04-21T18:17:26.118164058Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:17:45.605300124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:39264"
ts=2026-04-21T18:17:56.12058302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:17:57.725294417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:18:02.839948749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:33086"
ts=2026-04-21T18:18:26.123186799Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:18:45.608907151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:35018"
level=debug ts=2026-04-21T18:18:50.832135563Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:18:56.124890629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:19:02.843751672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:54844"
ts=2026-04-21T18:19:24.343743798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:54218"
ts=2026-04-21T18:19:26.126726988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:19:45.611834554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:58288"
level=debug ts=2026-04-21T18:19:47.783976338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:19:47.785877039Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:19:54.345830905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:39088"
ts=2026-04-21T18:19:56.128675141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:20:02.847903651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:45364"
level=error ts=2026-04-21T18:20:23.528341806Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:20:23.530279969Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:20:23.531994806Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:20:23.532042874Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:20:24.348191525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:51602"
ts=2026-04-21T18:20:26.130273426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e1f9390a 10.130.0.93:7946"
level=debug ts=2026-04-21T18:20:29.22189891Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:20:45.615546587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:47398"
ts=2026-04-21T18:20:56.132523256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e1f9390a 10.130.0.93:7946"
level=debug ts=2026-04-21T18:21:16.60502682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:21:26.134661089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e1f9390a 10.130.0.93:7946"
ts=2026-04-21T18:21:45.619581475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:46264"
ts=2026-04-21T18:21:56.136215033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e1f9390a 10.130.0.93:7946"
level=debug ts=2026-04-21T18:22:03.885201775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:22:26.138232977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:22:32.85532176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44538"
ts=2026-04-21T18:22:56.139793577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e1f9390a 10.130.0.93:7946"
level=debug ts=2026-04-21T18:23:02.46045819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:23:26.142256466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:23:54.85550906Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:23:54.857152669Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:23:56.144508937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:24:02.86067095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51702"
ts=2026-04-21T18:24:26.147106791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e1f9390a 10.130.0.93:7946"
ts=2026-04-21T18:24:32.863080867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:54216"
level=debug ts=2026-04-21T18:24:33.789316461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:24:56.148804703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:24:59.295929245Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e1f9390a (timeout reached)"
ts=2026-04-21T18:25:02.295369325Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e1f9390a has failed, no acks received"
ts=2026-04-21T18:25:02.865217609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44048"
ts=2026-04-21T18:25:08.335442845Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-e1f9390a as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T18:25:09.336339824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:46744"
ts=2026-04-21T18:25:09.3466699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:46748"
level=debug ts=2026-04-21T18:25:22.102951881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:25:26.150856702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97af508c 10.130.0.93:7946"
ts=2026-04-21T18:25:32.867607684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:45086"
ts=2026-04-21T18:25:56.152938796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97af508c 10.130.0.93:7946"
ts=2026-04-21T18:26:02.869901153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:58176"
level=debug ts=2026-04-21T18:26:11.619451304Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:26:26.155418746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:26:34.615052364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50190"
ts=2026-04-21T18:26:56.157330646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:27:02.873878518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49776"
ts=2026-04-21T18:27:04.619804748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:33910"
level=debug ts=2026-04-21T18:27:11.824197055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:27:26.159953921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:27:34.621604398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:35750"
ts=2026-04-21T18:27:56.162223684Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97af508c 10.130.0.93:7946"
level=debug ts=2026-04-21T18:27:57.735933049Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:27:57.737577679Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:28:02.87731241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60258"
ts=2026-04-21T18:28:24.374899389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:38800"
ts=2026-04-21T18:28:26.163743968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:28:32.879728201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:43654"
level=debug ts=2026-04-21T18:28:55.717478828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:28:56.16624369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:29:26.168275885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97af508c 10.130.0.93:7946"
level=debug ts=2026-04-21T18:29:45.829661515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:29:56.170487463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97af508c 10.130.0.93:7946"
ts=2026-04-21T18:30:04.628349154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:58830"
level=error ts=2026-04-21T18:30:23.528458907Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:30:23.530267921Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:30:23.531984994Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:30:23.532028725Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:30:24.380180399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:47978"
ts=2026-04-21T18:30:26.17238541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:30:31.686803354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:30:32.887482578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:37898"
ts=2026-04-21T18:30:56.174601708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:31:04.632098301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:45222"
level=debug ts=2026-04-21T18:31:13.921154723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:31:26.17710721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:31:32.891677515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:33676"
ts=2026-04-21T18:31:56.1789699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:32:07.964579952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:32:07.966310225Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:32:26.181228347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97af508c 10.130.0.93:7946"
ts=2026-04-21T18:32:56.183322668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:32:58.975936582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:33:26.185308987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97af508c 10.130.0.93:7946"
ts=2026-04-21T18:33:32.897979444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:46130"
level=debug ts=2026-04-21T18:33:49.936710877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:33:56.187498113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-97af508c 10.130.0.93:7946"
ts=2026-04-21T18:34:26.18918879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:34:47.102182646Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:34:49.296752349Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-97af508c (timeout reached)"
ts=2026-04-21T18:34:52.296738649Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-97af508c has failed, no acks received"
ts=2026-04-21T18:34:54.297053124Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-97af508c (timeout reached)"
ts=2026-04-21T18:34:56.191304642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:34:57.297255264Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-97af508c has failed, no acks received"
ts=2026-04-21T18:35:03.136749285Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-97af508c as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T18:35:20.754505632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:57172"
ts=2026-04-21T18:35:20.765508571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:57178"
ts=2026-04-21T18:35:24.394515576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:49886"
ts=2026-04-21T18:35:26.193616609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:35:31.163327606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:35:56.196082768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1bf9cd49 10.130.0.93:7946"
ts=2026-04-21T18:36:02.907151133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:33690"
level=debug ts=2026-04-21T18:36:24.842331437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:36:24.84394105Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:36:26.198476541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1bf9cd49 10.130.0.93:7946"
ts=2026-04-21T18:36:54.39876644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:38204"
ts=2026-04-21T18:36:56.200581746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:36:58.799097999Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:37:26.202507519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1bf9cd49 10.130.0.93:7946"
ts=2026-04-21T18:37:36.283828137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:37096"
level=debug ts=2026-04-21T18:37:48.128358133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:37:56.204534693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1bf9cd49 10.130.0.93:7946"
ts=2026-04-21T18:38:24.403954998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:55066"
ts=2026-04-21T18:38:26.206716433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:38:45.425975351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:38:56.208667049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:39:22.624263703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:39:26.210557674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1bf9cd49 10.130.0.93:7946"
ts=2026-04-21T18:39:56.212955114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:39:58.21370793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:39:58.215255885Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:40:23.532466691Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:40:23.53627948Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:40:23.538084725Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:40:23.538117121Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:40:26.214717015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:40:32.921042897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:33662"
ts=2026-04-21T18:40:36.293424952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:34208"
ts=2026-04-21T18:40:56.21740078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:40:57.826084267Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:41:02.923741644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44264"
ts=2026-04-21T18:41:24.412309295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:42666"
ts=2026-04-21T18:41:26.219220613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:41:32.925967796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:45232"
level=debug ts=2026-04-21T18:41:48.361198481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:41:56.221398854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:42:02.92820569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:57104"
level=debug ts=2026-04-21T18:42:23.485723792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:42:24.415238251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:57508"
ts=2026-04-21T18:42:26.223959303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1bf9cd49 10.130.0.93:7946"
ts=2026-04-21T18:42:36.298924856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:38942"
ts=2026-04-21T18:42:56.226573687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:43:20.5683238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:43:26.228441921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1bf9cd49 10.130.0.93:7946"
ts=2026-04-21T18:43:56.23046997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:44:01.694953813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:44:01.696601489Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:44:02.933939321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60560"
ts=2026-04-21T18:44:04.295427699Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1bf9cd49 (timeout reached)"
ts=2026-04-21T18:44:07.295864656Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1bf9cd49 has failed, no acks received"
ts=2026-04-21T18:44:16.752236205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:39922"
ts=2026-04-21T18:44:16.764273375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:39938"
ts=2026-04-21T18:44:26.232837556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:44:27.295954338Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-1bf9cd49 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T18:44:29.296028526Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1bf9cd49 (timeout reached)"
ts=2026-04-21T18:44:32.296567573Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1bf9cd49 has failed, no acks received"
level=debug ts=2026-04-21T18:44:46.076130149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:44:56.234963787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:45:02.306152173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:58188"
ts=2026-04-21T18:45:02.937201553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:41684"
ts=2026-04-21T18:45:26.237150651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
ts=2026-04-21T18:45:32.938877529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52660"
level=debug ts=2026-04-21T18:45:43.36499117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:45:56.239234792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:46:24.427417762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:44980"
ts=2026-04-21T18:46:26.241087143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
ts=2026-04-21T18:46:32.311224242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:57736"
level=debug ts=2026-04-21T18:46:45.549169333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:46:56.243465593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
ts=2026-04-21T18:47:02.313050923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:32836"
ts=2026-04-21T18:47:26.245308926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
ts=2026-04-21T18:47:32.314830166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:40850"
level=debug ts=2026-04-21T18:47:47.020909462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:47:54.433164824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:59850"
ts=2026-04-21T18:47:56.247964698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
ts=2026-04-21T18:48:24.435336774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:51684"
ts=2026-04-21T18:48:26.250286144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:48:43.246518711Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:48:43.248063467Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:48:56.252750191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:49:26.254406858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T18:49:31.164351927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:49:56.256377252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
ts=2026-04-21T18:50:02.322900406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:39932"
level=error ts=2026-04-21T18:50:23.527657109Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:50:23.529587958Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:50:23.531394728Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T18:50:23.531419083Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:50:24.4412572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:60670"
ts=2026-04-21T18:50:26.258550745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
level=debug ts=2026-04-21T18:50:27.264674495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:50:56.26022404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
ts=2026-04-21T18:51:02.953893181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:58886"
level=debug ts=2026-04-21T18:51:06.783907092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:51:26.262786654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:51:54.446491976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:45244"
ts=2026-04-21T18:51:56.264616324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ea376997 10.130.0.93:7946"
level=debug ts=2026-04-21T18:51:59.18767699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:52:02.328675288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50574"
ts=2026-04-21T18:52:02.95773321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:37652"
ts=2026-04-21T18:52:24.295312716Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ea376997 (timeout reached)"
ts=2026-04-21T18:52:26.266404566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:52:27.295280243Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ea376997 has failed, no acks received"
ts=2026-04-21T18:52:29.296363407Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ea376997 (timeout reached)"
ts=2026-04-21T18:52:32.295436833Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ea376997 has failed, no acks received"
ts=2026-04-21T18:52:32.959837115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:53636"
ts=2026-04-21T18:52:47.295821968Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-ea376997 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T18:52:49.296521785Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-ea376997 (timeout reached)"
ts=2026-04-21T18:52:49.755984647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:60350"
ts=2026-04-21T18:52:49.767335458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:60356"
ts=2026-04-21T18:52:52.296526372Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-ea376997 has failed, no acks received"
level=debug ts=2026-04-21T18:52:55.32885848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:52:55.330461159Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:52:56.26858313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:53:26.270941933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fd7d1a23 10.130.0.93:7946"
level=debug ts=2026-04-21T18:53:51.976147096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:53:56.273204891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:53:58.434241131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:48456"
ts=2026-04-21T18:54:26.275453314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fd7d1a23 10.130.0.93:7946"
ts=2026-04-21T18:54:32.967230353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:58682"
level=debug ts=2026-04-21T18:54:35.843478954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:54:56.277568636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:54:58.436780618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:34376"
ts=2026-04-21T18:55:26.280166309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T18:55:31.730555125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:55:32.970547735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35584"
ts=2026-04-21T18:55:54.458661022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:35338"
ts=2026-04-21T18:55:56.282180465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fd7d1a23 10.130.0.93:7946"
ts=2026-04-21T18:55:58.440743309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:39322"
level=debug ts=2026-04-21T18:56:19.250348483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:56:26.284193433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:56:32.974362989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:58046"
ts=2026-04-21T18:56:54.462385913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:54694"
ts=2026-04-21T18:56:56.286794199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fd7d1a23 10.130.0.93:7946"
ts=2026-04-21T18:56:58.44444051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:56884"
ts=2026-04-21T18:57:02.97677126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:50280"
level=debug ts=2026-04-21T18:57:05.936952695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T18:57:05.938791708Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T18:57:26.289394109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:57:32.979352573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39808"
ts=2026-04-21T18:57:54.465933821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:59166"
level=debug ts=2026-04-21T18:57:55.468905017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:57:56.291872131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T18:57:58.448036009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:33240"
ts=2026-04-21T18:58:24.467688686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:50358"
ts=2026-04-21T18:58:26.294399412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:58:32.983145403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:34116"
level=debug ts=2026-04-21T18:58:41.199710654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:58:56.296709825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:58:58.451411117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:42978"
level=debug ts=2026-04-21T18:59:16.216339058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:59:24.471135266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:32880"
ts=2026-04-21T18:59:26.299249606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T18:59:32.986616695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35316"
level=debug ts=2026-04-21T18:59:50.963676687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T18:59:56.301180622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:00:09.296156004Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-fd7d1a23 (timeout reached)"
ts=2026-04-21T19:00:12.295128493Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-fd7d1a23 has failed, no acks received"
ts=2026-04-21T19:00:19.296166743Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-fd7d1a23 (timeout reached)"
ts=2026-04-21T19:00:22.295863105Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-fd7d1a23 has failed, no acks received"
level=error ts=2026-04-21T19:00:23.527582869Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T19:00:23.529736773Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T19:00:23.531549926Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T19:00:23.531575113Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:00:24.475152631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:56140"
ts=2026-04-21T19:00:26.303268981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:00:28.13725829Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-fd7d1a23 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-21T19:00:45.575464654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T19:00:45.577412816Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:00:47.770032741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50296"
ts=2026-04-21T19:00:47.781715279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50308"
ts=2026-04-21T19:00:54.477130481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:60296"
ts=2026-04-21T19:00:56.305113236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:01:24.479708995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:60886"
ts=2026-04-21T19:01:26.306800479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T19:01:36.661809142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:01:56.309373267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0722c459 10.130.0.93:7946"
ts=2026-04-21T19:02:02.994512226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38970"
ts=2026-04-21T19:02:15.478148234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:46604"
ts=2026-04-21T19:02:26.311366904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0722c459 10.130.0.93:7946"
ts=2026-04-21T19:02:32.996848253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44514"
level=debug ts=2026-04-21T19:02:33.105667679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:02:54.485583437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:35952"
ts=2026-04-21T19:02:56.313801887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0722c459 10.130.0.93:7946"
level=debug ts=2026-04-21T19:03:23.572564683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:03:26.315762305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:03:45.483108715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:38150"
ts=2026-04-21T19:03:56.318166852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T19:04:06.058481816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:04:15.484810012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:39334"
ts=2026-04-21T19:04:24.490212661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:57956"
ts=2026-04-21T19:04:26.320739818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T19:04:49.596669341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T19:04:49.598471562Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:04:56.323121471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T19:05:26.325043538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0722c459 10.130.0.93:7946"
level=debug ts=2026-04-21T19:05:31.54371675Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:05:45.489421938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:60510"
ts=2026-04-21T19:05:56.327129705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0722c459 10.130.0.93:7946"
ts=2026-04-21T19:06:03.008285275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:32894"
level=debug ts=2026-04-21T19:06:16.626391526Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:06:26.329734647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-0722c459 10.130.0.93:7946"
ts=2026-04-21T19:06:45.493718452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:54640"
level=debug ts=2026-04-21T19:06:52.58041572Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:06:56.332192424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:07:04.29548634Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0722c459 (timeout reached)"
ts=2026-04-21T19:07:07.295137654Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0722c459 has failed, no acks received"
ts=2026-04-21T19:07:14.295988672Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0722c459 (timeout reached)"
ts=2026-04-21T19:07:17.295667142Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0722c459 has failed, no acks received"
ts=2026-04-21T19:07:24.2954061Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-0722c459 (timeout reached)"
ts=2026-04-21T19:07:24.500111578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:38302"
ts=2026-04-21T19:07:26.334114165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:07:27.295802829Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-0722c459 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-21T19:07:27.295845136Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-0722c459 has failed, no acks received"
ts=2026-04-21T19:07:33.011894292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:41166"
level=debug ts=2026-04-21T19:07:44.717892927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:07:56.336576057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:08:03.014433827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:43520"
ts=2026-04-21T19:08:20.742894953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:48504"
ts=2026-04-21T19:08:20.753022303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:48516"
ts=2026-04-21T19:08:26.338842938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T19:08:28.619316131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T19:08:28.620923255Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:08:56.340577928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e43e322 10.130.0.93:7946"
ts=2026-04-21T19:09:03.01804849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51346"
ts=2026-04-21T19:09:11.150741232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:38356"
level=debug ts=2026-04-21T19:09:19.410789283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:09:26.34262639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:09:54.508044209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:52286"
ts=2026-04-21T19:09:56.344830925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T19:10:08.783266769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:10:11.154495229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:33424"
level=error ts=2026-04-21T19:10:23.526827529Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T19:10:23.52875533Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T19:10:23.530567047Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-21T19:10:23.530593103Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:10:24.509571179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:57654"
ts=2026-04-21T19:10:26.34741046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T19:10:54.511411357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:37832"
ts=2026-04-21T19:10:56.349387395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T19:11:10.252987531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:11:11.157383922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:47416"
ts=2026-04-21T19:11:26.351319866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T19:11:48.521482161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:11:56.353328919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
ts=2026-04-21T19:12:26.3558653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e43e322 10.130.0.93:7946"
level=debug ts=2026-04-21T19:12:43.303398368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T19:12:43.305291935Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:12:56.358028008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T19:13:11.164230932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:53268"
ts=2026-04-21T19:13:26.360103198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e43e322 10.130.0.93:7946"
ts=2026-04-21T19:13:33.030896826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60758"
ts=2026-04-21T19:13:41.16621113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:34052"
level=debug ts=2026-04-21T19:13:43.639526242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:13:56.362356838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e43e322 10.130.0.93:7946"
level=debug ts=2026-04-21T19:14:24.46761378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:14:26.364548841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-8e43e322 10.130.0.93:7946"
ts=2026-04-21T19:14:33.033897976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:55736"
ts=2026-04-21T19:14:54.52284298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:35156"
ts=2026-04-21T19:14:56.367131945Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T19:15:09.296285669Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8e43e322 (timeout reached)"
level=debug ts=2026-04-21T19:15:12.195077827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:15:12.29636979Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8e43e322 has failed, no acks received"
ts=2026-04-21T19:15:24.296527697Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8e43e322 (timeout reached)"
ts=2026-04-21T19:15:24.525432366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:41786"
ts=2026-04-21T19:15:26.369242275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T19:15:27.295955098Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8e43e322 has failed, no acks received"
ts=2026-04-21T19:15:29.296515436Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-8e43e322 (timeout reached)"
ts=2026-04-21T19:15:32.29705888Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-8e43e322 has failed, no acks received"
ts=2026-04-21T19:15:54.527820373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:41568"
ts=2026-04-21T19:15:56.377064909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
level=debug ts=2026-04-21T19:16:03.699298142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:16:26.379800057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T19:16:33.039408575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:46032"
ts=2026-04-21T19:16:54.531664597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.88:45988"
ts=2026-04-21T19:16:56.381501472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"
level=debug ts=2026-04-21T19:17:03.413402015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T19:17:03.414966608Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T19:17:26.384167767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-b52xc-e08c6741 10.130.0.89:7946"
ts=2026-04-21T19:17:43.754285242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:42232"
ts=2026-04-21T19:17:43.765302673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:42234"
level=debug ts=2026-04-21T19:17:56.015444233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T19:17:56.386107425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-bw6qm-cebba9b0 10.130.0.88:7946"