level=info ts=2026-04-19T19:46:39.7233932Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-19T19:46:39.723424364Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-19T19:46:39.72523241Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-19T19:46:39.725571631Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T19:46:39.725657183Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.74 inf=eth0
level=debug ts=2026-04-19T19:46:39.726156333Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T19:46:39.726216431Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.74 inf=eth0
level=info ts=2026-04-19T19:46:39.726311103Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-74d8e765
ts=2026-04-19T19:46:39.726403002Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-19T19:46:39.726421164Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.74 advertisePort=7946
level=debug ts=2026-04-19T19:46:39.726449478Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.74 advertisePort=7946
level=info ts=2026-04-19T19:46:39.726608459Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-19T19:46:39.734965847Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T19:46:39.735010587Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T19:46:39.735025573Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.419907ms
level=error ts=2026-04-19T19:46:39.735036738Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-19T19:46:39.735053665Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-19T19:46:39.738075676Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:46:39.738092859Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T19:46:39.742160514Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-19T19:46:39.742209034Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-19T19:46:39.742228114Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-19T19:46:39.742237634Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-19T19:46:39.742267646Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-19T19:46:39.742246663Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-19T19:46:39.74228401Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-19T19:46:39.742565587Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-19T19:46:39.742577784Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-19T19:46:39.74258465Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-19T19:46:39.742640419Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-19T19:46:39.742666469Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-19T19:46:39.742697621Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-19T19:46:39.74291242Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-19T19:46:39.742926634Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-19T19:46:39.742961557Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-19T19:46:40.743208917Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-19T19:46:40.87747926Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-19T19:46:40.877535945Z caller=loki.go:599 msg="Loki started" startup_time=1.219992022s
ts=2026-04-19T19:46:41.449229354Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:46:41.449253676Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:46:44.170608667Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:46:44.170634405Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-19T19:46:45.878016814Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-19T19:46:45.878093326Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-19T19:46:49.40143707Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T19:46:49.401460528Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T19:46:52.157907957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:47:02.286631089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50736"
ts=2026-04-19T19:47:02.752588933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47586"
ts=2026-04-19T19:47:03.857810925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.74:7946"
ts=2026-04-19T19:47:03.857914774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.74:39222"
ts=2026-04-19T19:47:03.861049621Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.73:7946: dial tcp 10.128.0.73:7946: connect: connection refused"
ts=2026-04-19T19:47:03.862785385Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
ts=2026-04-19T19:47:03.864590738Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
level=info ts=2026-04-19T19:47:03.864603085Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=24.129542838s
ts=2026-04-19T19:47:04.728516627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:40706"
level=debug ts=2026-04-19T19:47:10.241962813Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T19:47:26.591008605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:47:28.418425826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:47:33.763061743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49922"
level=debug ts=2026-04-19T19:47:45.915766382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:47:58.421604373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:48:01.934954243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:37520"
ts=2026-04-19T19:48:20.761601016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:60338"
level=debug ts=2026-04-19T19:48:21.332595584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T19:48:21.334541696Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:48:28.424405848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:48:31.938113867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:49788"
ts=2026-04-19T19:48:33.769296477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57660"
ts=2026-04-19T19:48:50.764331591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:58884"
ts=2026-04-19T19:48:58.427416519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T19:49:01.941624538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:55948"
level=debug ts=2026-04-19T19:49:03.206521909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:49:28.430661513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T19:49:33.774813973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48268"
ts=2026-04-19T19:49:50.770028499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:59526"
ts=2026-04-19T19:49:58.433678811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T19:50:03.573377722Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:50:28.43660136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:50:50.775604289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:55016"
ts=2026-04-19T19:50:58.439266082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:51:01.949535431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:35354"
level=debug ts=2026-04-19T19:51:02.649107425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:51:03.782588183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38520"
ts=2026-04-19T19:51:28.442530205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T19:51:56.782194725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T19:51:58.445041759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T19:52:01.954137497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:37270"
ts=2026-04-19T19:52:28.447687117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:52:31.957144138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:38546"
ts=2026-04-19T19:52:33.789557564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55102"
ts=2026-04-19T19:52:58.450475157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T19:53:03.505493443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T19:53:03.507173834Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:53:03.792497243Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38530"
ts=2026-04-19T19:53:28.454119255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T19:53:31.962592918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:43242"
level=debug ts=2026-04-19T19:53:46.182061853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:53:58.457290738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T19:54:20.78911027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:45494"
ts=2026-04-19T19:54:28.459851239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T19:54:46.828406678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:54:58.462601701Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:55:28.465424908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:55:33.802776625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52596"
level=debug ts=2026-04-19T19:55:34.07455848Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:55:58.468721705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T19:56:01.973620428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:40496"
ts=2026-04-19T19:56:03.805894397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54954"
level=debug ts=2026-04-19T19:56:14.050868511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:56:28.472037825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=info ts=2026-04-19T19:56:45.87818323Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-19T19:56:45.883423207Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:56:45.885507339Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:56:45.887646243Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T19:56:45.887674935Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T19:56:45.887686758Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-19T19:56:58.475183009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T19:56:58.919610146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T19:56:58.921171477Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T19:57:01.978343274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:35186"
ts=2026-04-19T19:57:28.478136089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T19:57:33.81320839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49728"
level=debug ts=2026-04-19T19:57:47.810186246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:57:58.481200223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:58:20.803876678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:51570"
ts=2026-04-19T19:58:28.484067204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:58:31.984890357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:57584"
level=debug ts=2026-04-19T19:58:43.784699187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:58:58.487003634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T19:59:28.490045587Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T19:59:36.150363094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T19:59:50.809830558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:38714"
ts=2026-04-19T19:59:58.493321933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:00:01.991803181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:48972"
ts=2026-04-19T20:00:20.81234274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:42768"
ts=2026-04-19T20:00:28.495737309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T20:00:32.247249313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:00:33.826227516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60594"
ts=2026-04-19T20:00:58.498544718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:01:01.99527615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:47884"
level=debug ts=2026-04-19T20:01:13.911710717Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:01:13.913309243Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:01:28.501234749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:01:58.503960982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T20:02:03.171846817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:02:28.506445054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:02:50.821988612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:53902"
ts=2026-04-19T20:02:58.508963091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T20:03:00.621445418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:03:03.838566739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52294"
ts=2026-04-19T20:03:20.824890223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:55168"
ts=2026-04-19T20:03:28.512128585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:03:35.025277348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:03:50.827436492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:55928"
ts=2026-04-19T20:03:58.51477405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:04:28.517739015Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T20:04:30.819691935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:04:58.520818426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:05:02.01272544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:58868"
ts=2026-04-19T20:05:20.834385538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:51626"
level=debug ts=2026-04-19T20:05:26.324053965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:05:26.325920736Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:05:28.52351897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:05:58.526712434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T20:06:04.779949095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:06:20.838944098Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:55486"
ts=2026-04-19T20:06:28.529459569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=error ts=2026-04-19T20:06:45.892749596Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:06:45.894660019Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:06:45.896333138Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:06:45.896361542Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:06:50.84161107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:37672"
ts=2026-04-19T20:06:58.532327527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:06:59.132968967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:07:28.535391561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:07:56.583083869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:07:58.538371231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:08:02.022215846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:54412"
ts=2026-04-19T20:08:20.848937765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:56530"
ts=2026-04-19T20:08:28.541377944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:08:32.024734144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:57986"
level=debug ts=2026-04-19T20:08:33.215855004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:08:50.851217981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:36870"
ts=2026-04-19T20:08:58.544252428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:09:10.446233847Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:09:10.447664106Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:09:28.546831523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:09:32.029322441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:37018"
ts=2026-04-19T20:09:58.549747508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:10:09.109946048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:10:20.857146663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:35756"
ts=2026-04-19T20:10:28.55230895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:10:33.873652833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:43988"
ts=2026-04-19T20:10:50.860206947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:53988"
ts=2026-04-19T20:10:58.555505624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:11:08.619491968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:11:28.558349816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:11:33.882129416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34386"
ts=2026-04-19T20:11:58.561025588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:12:02.040514698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:40344"
ts=2026-04-19T20:12:03.887179755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60864"
level=debug ts=2026-04-19T20:12:07.501541877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:12:28.56362881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:12:42.107728215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:12:58.569841519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:13:03.893170523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45996"
ts=2026-04-19T20:13:28.573106031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:13:32.048900355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:58302"
ts=2026-04-19T20:13:33.896149665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55280"
level=debug ts=2026-04-19T20:13:36.359096037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:13:36.360859749Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:13:58.575955769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:14:02.051459363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:59276"
ts=2026-04-19T20:14:03.899189535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33268"
level=debug ts=2026-04-19T20:14:17.547014231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:14:20.877763862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:41732"
ts=2026-04-19T20:14:28.578844492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:14:32.053773084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:42476"
ts=2026-04-19T20:14:58.581271836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:15:17.935433144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:15:28.584074384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:15:32.059013655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:41948"
ts=2026-04-19T20:15:58.586994347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:16:03.169672238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:16:28.589895264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=error ts=2026-04-19T20:16:45.893592415Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:16:45.895614168Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:16:45.897390976Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:16:45.897420966Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-19T20:16:58.171272705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:16:58.592432071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:17:03.912184604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39816"
ts=2026-04-19T20:17:20.891318513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50492"
ts=2026-04-19T20:17:28.595089282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:17:32.066696943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:42212"
level=debug ts=2026-04-19T20:17:33.75202017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:17:33.75380756Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:17:58.59816449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:18:02.069426147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:45154"
ts=2026-04-19T20:18:03.917577398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48490"
ts=2026-04-19T20:18:20.896120527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50110"
ts=2026-04-19T20:18:28.601105955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:18:33.433728009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:18:50.898546288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:46296"
ts=2026-04-19T20:18:58.604484335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:19:20.901783112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:46748"
ts=2026-04-19T20:19:28.607954987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T20:19:28.964546559Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:19:32.076860533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:60896"
ts=2026-04-19T20:19:50.904934425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:49224"
ts=2026-04-19T20:19:58.610354431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T20:20:03.971623144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:20:28.612664091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:20:33.929734187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46734"
level=debug ts=2026-04-19T20:20:43.565322973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:20:50.910504235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:52720"
ts=2026-04-19T20:20:58.615892595Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:21:02.082937894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:60192"
ts=2026-04-19T20:21:28.618889854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:21:39.588400142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:21:39.590358232Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:21:50.913774958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:50574"
ts=2026-04-19T20:21:58.62154577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:22:02.087043141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:52044"
ts=2026-04-19T20:22:28.624293655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:22:30.024123315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:22:58.627236968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:23:02.09473071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:34564"
level=debug ts=2026-04-19T20:23:20.971002317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:23:28.629776423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:23:32.098193657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:59120"
ts=2026-04-19T20:23:50.922655534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:51832"
ts=2026-04-19T20:23:58.633217876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:24:08.655038239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:24:28.636574608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:24:43.556845092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:24:58.639876121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:25:27.769097465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:25:27.770768271Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:25:28.642404995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:25:58.645370216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:26:25.998034357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:26:28.647833986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=error ts=2026-04-19T20:26:45.891696371Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:26:45.893667298Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:26:45.895324425Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:26:45.895348914Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:26:58.650593174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:27:19.627229723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:27:28.653616314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:27:32.111878382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:57118"
ts=2026-04-19T20:27:50.938433183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:56648"
ts=2026-04-19T20:27:58.656329995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:28:02.114656825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:35658"
level=debug ts=2026-04-19T20:28:09.988058721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:28:20.941245082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:58186"
ts=2026-04-19T20:28:28.661460134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:28:50.94391104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:44256"
ts=2026-04-19T20:28:58.664197995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:29:02.11989758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:46142"
level=debug ts=2026-04-19T20:29:09.5564578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:29:28.667211428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:29:45.498315507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:29:45.500930453Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:29:58.670370892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:30:02.124143733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:57166"
ts=2026-04-19T20:30:28.678667912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:30:32.126832906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:41694"
level=debug ts=2026-04-19T20:30:44.062516004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:30:58.683569666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:31:02.131951241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:33928"
ts=2026-04-19T20:31:20.953365941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:34592"
ts=2026-04-19T20:31:28.686406221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:31:32.134539789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:42094"
level=debug ts=2026-04-19T20:31:45.232923936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:31:50.956079261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:36406"
ts=2026-04-19T20:31:58.689502146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:32:03.980061235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39546"
ts=2026-04-19T20:32:28.69267481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:32:29.273062468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:32:58.695636489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:33:07.010290744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:33:28.69861314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:33:58.70132869Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:34:03.991539738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:56714"
level=debug ts=2026-04-19T20:34:07.739618624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:34:07.741576602Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:34:28.704470299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:34:50.966164341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:48050"
ts=2026-04-19T20:34:58.707276401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:34:59.701469585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:35:03.996283491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42270"
ts=2026-04-19T20:35:28.710389606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:35:32.153698547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:48026"
level=debug ts=2026-04-19T20:35:52.016693542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:35:58.713280631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:36:28.715936057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:36:32.158865345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:40328"
level=debug ts=2026-04-19T20:36:34.751082144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:36:45.892408368Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:36:45.894754988Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:36:45.897233255Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:36:45.897262161Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:36:58.718867647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:37:02.162049982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:34246"
level=debug ts=2026-04-19T20:37:11.014575687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:37:20.976620359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:53376"
ts=2026-04-19T20:37:28.722012389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
ts=2026-04-19T20:37:32.165158604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:50382"
ts=2026-04-19T20:37:50.979526404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:45440"
ts=2026-04-19T20:37:58.724697571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:38:11.869197007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:38:11.871828983Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:38:20.982949899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:46124"
ts=2026-04-19T20:38:28.727331361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:38:58.729993936Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:39:11.283737459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:39:20.987267007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:54784"
ts=2026-04-19T20:39:28.733151182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:39:54.361925994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:39:58.736508521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:40:04.02017151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59034"
ts=2026-04-19T20:40:28.738962077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:40:34.023776533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60488"
level=debug ts=2026-04-19T20:40:48.296428252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:40:58.742170837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:41:02.180547026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:48788"
ts=2026-04-19T20:41:20.993374571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:58240"
ts=2026-04-19T20:41:28.744784383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:41:43.059683335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:41:58.747782094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:42:28.750809946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"
level=debug ts=2026-04-19T20:42:29.696933368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:42:29.698839205Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:42:32.188126893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:43650"
ts=2026-04-19T20:42:50.998816692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:53046"
ts=2026-04-19T20:42:58.753794531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:43:19.10114973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:43:21.001681639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:37362"
ts=2026-04-19T20:43:28.756756408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
level=debug ts=2026-04-19T20:43:58.216103006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:43:58.759990576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:44:28.763114873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:44:51.008890725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:47272"
level=debug ts=2026-04-19T20:44:56.016119829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:44:58.766024792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
ts=2026-04-19T20:45:28.768821498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:45:31.934155234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:45:32.202206095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.95:36112"
ts=2026-04-19T20:45:34.047142411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34738"
ts=2026-04-19T20:45:58.772013357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ggqfx-b6f692b5 10.130.0.93:7946"
ts=2026-04-19T20:46:28.775078888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:46:33.251485459Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T20:46:33.253250557Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:46:34.052502259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40196"
level=error ts=2026-04-19T20:46:45.892100722Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:46:45.894059316Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:46:45.895840666Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T20:46:45.895865199Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T20:46:58.778434783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-41be47ad 10.128.0.74:7946"
level=debug ts=2026-04-19T20:47:17.387064507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T20:47:21.017033922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.93:59670"
ts=2026-04-19T20:47:28.781817724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-jxmr7-12e13bbd 10.130.0.95:7946"