level=info ts=2026-05-15T11:36:49.997015156Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-15T11:36:49.997042096Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-15T11:36:49.9979066Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-15T11:36:49.998509724Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-15T11:36:49.998610281Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.73 inf=eth0
level=debug ts=2026-05-15T11:36:49.999992247Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-15T11:36:50.000071397Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.73 inf=eth0
level=info ts=2026-05-15T11:36:50.000122797Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-1ece0b2e
ts=2026-05-15T11:36:50.000208146Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-15T11:36:50.000224106Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.73 advertisePort=7946
level=debug ts=2026-05-15T11:36:50.00024851Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.73 advertisePort=7946
level=info ts=2026-05-15T11:36:50.00041485Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-15T11:36:50.005857653Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-15T11:36:50.00587926Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-15T11:36:50.005893259Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.480998ms
level=error ts=2026-05-15T11:36:50.005904824Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-15T11:36:50.005914836Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-15T11:36:50.009384857Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-15T11:36:50.009407199Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-15T11:36:50.017537921Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-15T11:36:50.017606054Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-05-15T11:36:50.017618538Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-15T11:36:50.017663775Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-15T11:36:50.017681308Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-15T11:36:50.017703475Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-15T11:36:50.01804055Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-05-15T11:36:50.018056094Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-15T11:36:50.018063434Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-05-15T11:36:50.018068338Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-15T11:36:50.0181444Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-15T11:36:50.018148273Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-15T11:36:50.018187318Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-15T11:36:50.018383499Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-15T11:36:50.018398736Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-15T11:36:50.01841708Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-15T11:36:51.018621028Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-15T11:36:51.168938238Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-15T11:36:51.169010468Z caller=loki.go:615 msg="Loki started" startup_time=1.244349425s
ts=2026-05-15T11:36:51.243474253Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-15T11:36:51.243497208Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T11:36:53.251965006Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-15T11:36:53.251986149Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-15T11:36:56.169659749Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-15T11:36:56.169714428Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-15T11:36:57.539834913Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-15T11:36:57.539859844Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-15T11:37:03.985703612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T11:37:10.545312391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42570"
ts=2026-05-15T11:37:10.621377839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55634"
ts=2026-05-15T11:37:11.534680266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.71:7946"
ts=2026-05-15T11:37:11.535798307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.69:7946"
ts=2026-05-15T11:37:11.537722704Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.65:7946: dial tcp 10.129.0.65:7946: connect: connection refused"
ts=2026-05-15T11:37:11.537849154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.73:7946"
ts=2026-05-15T11:37:11.537937706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33108"
ts=2026-05-15T11:37:11.539180994Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.74:7946: dial tcp 10.129.0.74:7946: connect: connection refused"
ts=2026-05-15T11:37:11.5398742Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.68:7946: dial tcp 10.129.0.68:7946: connect: connection refused"
level=info ts=2026-05-15T11:37:11.539886278Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=21.533955777s
ts=2026-05-15T11:37:14.172049325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:58636"
level=debug ts=2026-05-15T11:37:21.534863983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T11:37:36.540748808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
level=debug ts=2026-05-15T11:37:37.204378136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-15T11:38:06.27672336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T11:38:06.542630631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:38:07.651891532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40716"
ts=2026-05-15T11:38:11.294568922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33910"
ts=2026-05-15T11:38:18.065646835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36300"
ts=2026-05-15T11:38:36.544085812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T11:38:39.640990486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-15T11:38:39.642606717Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:39:06.545512673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T11:39:24.229975293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T11:39:36.546453664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:39:37.656006983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40188"
ts=2026-05-15T11:39:41.298994694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:50598"
ts=2026-05-15T11:39:48.069235212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:36394"
ts=2026-05-15T11:40:06.547750724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:40:11.300582809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41866"
ts=2026-05-15T11:40:18.070498829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:32994"
level=debug ts=2026-05-15T11:40:29.315803328Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T11:40:36.549279546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:41:06.55095541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:41:11.303485741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:58146"
level=debug ts=2026-05-15T11:41:24.632654582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T11:41:36.55230242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:41:37.661930959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:51162"
ts=2026-05-15T11:42:06.553348512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:42:18.07630166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58558"
level=debug ts=2026-05-15T11:42:28.809987965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-15T11:42:36.554880753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:43:06.555666908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:43:11.308930921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53250"
level=debug ts=2026-05-15T11:43:30.884556684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-15T11:43:30.886186679Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:43:36.556945818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:43:48.080375348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52544"
ts=2026-05-15T11:44:06.558020112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:44:18.082193382Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:57282"
level=debug ts=2026-05-15T11:44:24.667347545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:44:36.55914407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:44:37.670828096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39386"
level=debug ts=2026-05-15T11:45:04.730929228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:45:06.560285149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:45:11.314457548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:47384"
ts=2026-05-15T11:45:36.561901412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T11:45:39.850744736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:45:41.315723057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54296"
ts=2026-05-15T11:46:06.563729506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:46:11.316941794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55218"
ts=2026-05-15T11:46:18.087281743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44720"
level=debug ts=2026-05-15T11:46:27.497704515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:46:36.564971014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:46:37.676195265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42228"
level=info ts=2026-05-15T11:46:56.170727506Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-15T11:46:56.174503623Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:46:56.176388091Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:46:56.178133871Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:46:56.178162707Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-15T11:46:56.178175812Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-15T11:47:06.566423972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:47:07.67799913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54134"
level=debug ts=2026-05-15T11:47:27.416669817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:47:27.418274461Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:47:36.567956785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T11:48:06.201202926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:48:06.568778282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:48:07.681017881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36960"
ts=2026-05-15T11:48:36.569735547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:48:37.682279783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:56392"
ts=2026-05-15T11:48:41.324378748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:58962"
level=debug ts=2026-05-15T11:48:59.218682569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:49:06.570974568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:49:11.325253016Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:51942"
ts=2026-05-15T11:49:18.096729174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52016"
ts=2026-05-15T11:49:36.57258439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:49:37.685597843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:40034"
level=debug ts=2026-05-15T11:49:39.135201027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:50:06.57433659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:50:11.328158315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40102"
ts=2026-05-15T11:50:36.576144602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
level=debug ts=2026-05-15T11:50:40.390859305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:50:41.329931809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:48500"
ts=2026-05-15T11:51:06.57793998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:51:11.331365128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:40098"
ts=2026-05-15T11:51:18.103049446Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:45630"
level=debug ts=2026-05-15T11:51:36.100354614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:51:36.101940877Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:51:36.579426713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:51:37.690911752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:37028"
ts=2026-05-15T11:51:41.332399606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:58090"
ts=2026-05-15T11:52:06.581153568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:52:11.33428258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:34770"
level=debug ts=2026-05-15T11:52:30.811297331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:52:36.582608872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:53:06.583477655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:53:11.337877257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:54966"
level=debug ts=2026-05-15T11:53:30.927267771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:53:36.585230803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:53:41.339135757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:49534"
ts=2026-05-15T11:54:06.586901307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T11:54:19.558472779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:54:36.587903387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:54:48.112784802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42646"
ts=2026-05-15T11:55:06.589517654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T11:55:19.994571714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:55:36.5906314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:55:37.703050447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:47812"
ts=2026-05-15T11:55:48.115510263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38926"
ts=2026-05-15T11:56:06.592079654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:56:18.116922913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:58552"
level=debug ts=2026-05-15T11:56:19.393507248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T11:56:19.395124257Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:56:36.593132248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T11:56:48.118239791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55898"
level=error ts=2026-05-15T11:56:56.182510399Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:56:56.184426912Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:56:56.186489966Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T11:56:56.186519939Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T11:57:06.594888325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T11:57:18.151436102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:57:36.596494168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:57:37.708656874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39484"
level=debug ts=2026-05-15T11:58:05.71196525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:58:06.597979721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:58:07.710759672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43900"
ts=2026-05-15T11:58:36.599375835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
level=debug ts=2026-05-15T11:58:55.947602803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T11:59:06.600670806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T11:59:36.60176499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T11:59:37.714985441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33468"
ts=2026-05-15T11:59:48.12835962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42066"
level=debug ts=2026-05-15T11:59:57.763884383Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:00:06.603417533Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:00:18.130017477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60204"
ts=2026-05-15T12:00:36.605107651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:00:37.717715028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:42216"
ts=2026-05-15T12:00:48.13126108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:38316"
level=debug ts=2026-05-15T12:00:55.851349362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:00:55.853157523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:01:06.606325811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:01:36.608020445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T12:01:50.58940955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:02:06.609921246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:02:36.611767637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
level=debug ts=2026-05-15T12:02:41.423427856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:03:06.612844494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:03:11.36711349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:39102"
ts=2026-05-15T12:03:18.138735958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:42350"
ts=2026-05-15T12:03:36.614291319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
level=debug ts=2026-05-15T12:03:39.557883739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:04:06.616091852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:04:07.728524861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:47382"
ts=2026-05-15T12:04:11.370137979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:43298"
level=debug ts=2026-05-15T12:04:32.306986953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:04:36.61793654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:05:06.619249271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:05:07.731275948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:59894"
ts=2026-05-15T12:05:11.373643982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41328"
level=debug ts=2026-05-15T12:05:14.848185971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:05:14.849907265Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:05:36.620859988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T12:05:56.303186869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:06:06.622717613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:06:07.734505889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:48394"
ts=2026-05-15T12:06:36.624312618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T12:06:47.152403771Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:06:56.180737578Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:06:56.183019071Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:06:56.184753985Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:06:56.184783315Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:07:06.625713509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:07:07.737108717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:39776"
level=debug ts=2026-05-15T12:07:30.213334168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:07:36.627443279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:07:48.151207535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:35004"
ts=2026-05-15T12:08:06.628934894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:08:07.740599167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33516"
level=debug ts=2026-05-15T12:08:15.084672672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:08:36.630222394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:08:37.742180805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:41814"
ts=2026-05-15T12:08:41.384029783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52008"
level=debug ts=2026-05-15T12:09:04.008974567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:09:04.010792486Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:09:06.631511536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:09:11.385428627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:60132"
ts=2026-05-15T12:09:36.632737649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T12:10:03.517854326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:10:06.634164685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:10:07.746095437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45236"
ts=2026-05-15T12:10:36.635690647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T12:10:46.943929473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:11:06.63709804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T12:11:34.237559909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:11:36.638457916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:12:06.63930043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
level=debug ts=2026-05-15T12:12:30.315589983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:12:36.640801526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:12:48.164472686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52552"
ts=2026-05-15T12:13:06.642448085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T12:13:24.530940207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:13:24.53274045Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:13:36.644291035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:13:41.399209667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55314"
ts=2026-05-15T12:14:06.645370046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:14:11.400905583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:47756"
level=debug ts=2026-05-15T12:14:13.573780282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:14:36.646583357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:14:48.169618232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:48686"
level=debug ts=2026-05-15T12:15:05.025878362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:15:06.648386407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:15:11.404178918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38306"
ts=2026-05-15T12:15:18.170679874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:55844"
ts=2026-05-15T12:15:36.650207488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
level=debug ts=2026-05-15T12:16:02.593885219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:16:06.651894822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:16:36.65289409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T12:16:54.592914596Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:16:56.180841613Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:16:56.182565516Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:16:56.184149397Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:16:56.184170045Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:17:06.653887581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:17:18.175747354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:34634"
level=debug ts=2026-05-15T12:17:32.496879362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:17:32.498583806Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:17:36.65555767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:17:48.176884953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:46538"
ts=2026-05-15T12:18:06.657374937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T12:18:23.969246482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:18:36.658941046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:19:06.660257974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:19:07.770334023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:35964"
ts=2026-05-15T12:19:18.181566493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:47118"
level=debug ts=2026-05-15T12:19:22.670823931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:19:36.661390829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:19:37.772225263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53144"
ts=2026-05-15T12:20:06.662781695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:20:07.773528356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52472"
level=debug ts=2026-05-15T12:20:09.429610874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:20:36.664243666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T12:21:00.528654078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:21:06.665264914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:21:07.776761245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:44528"
ts=2026-05-15T12:21:11.421751826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:57884"
ts=2026-05-15T12:21:36.666426642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:21:41.422899279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:41112"
ts=2026-05-15T12:21:48.189031906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:41808"
level=debug ts=2026-05-15T12:21:51.866976614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:21:51.868713338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:22:06.667499911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:22:36.669312221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:22:37.781139568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:52988"
level=debug ts=2026-05-15T12:22:41.316688119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:23:06.670502814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:23:11.425799469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53422"
level=debug ts=2026-05-15T12:23:16.563185028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:23:36.671633245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T12:23:57.368939814Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:24:06.673360205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:24:07.785507697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43152"
ts=2026-05-15T12:24:36.674087275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:24:37.787109577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:54334"
ts=2026-05-15T12:24:48.198539313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56342"
level=debug ts=2026-05-15T12:24:51.62404769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:25:06.675056425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:25:07.788621136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:33444"
ts=2026-05-15T12:25:36.67622479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T12:25:44.770062801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:25:44.771618534Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:25:48.200665001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:50452"
ts=2026-05-15T12:26:06.677520344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:26:07.790675787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43646"
level=debug ts=2026-05-15T12:26:20.90242025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:26:36.678879453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:26:48.203044384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:52838"
level=error ts=2026-05-15T12:26:56.181021069Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:26:56.182647312Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:26:56.18432748Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:26:56.184349645Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-15T12:26:58.181846784Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:27:06.680085357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:27:07.793447212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:53832"
ts=2026-05-15T12:27:11.437790538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53686"
ts=2026-05-15T12:27:36.68122916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T12:27:53.31308418Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:28:06.682667439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:28:36.683909994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:28:37.796939087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:43574"
level=debug ts=2026-05-15T12:28:54.446848416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:29:06.68568226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:29:11.443674196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:47206"
ts=2026-05-15T12:29:18.209662495Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44402"
ts=2026-05-15T12:29:36.687247565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:29:37.800413133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:47020"
ts=2026-05-15T12:29:41.445152921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:38136"
level=debug ts=2026-05-15T12:29:44.947564922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:29:44.949045153Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:30:06.688780763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:30:11.446467827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53326"
ts=2026-05-15T12:30:36.690138237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:30:37.802113166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:50918"
level=debug ts=2026-05-15T12:30:47.100983668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:30:48.214953616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:44476"
ts=2026-05-15T12:31:06.69150777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T12:31:26.20943857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:31:36.692742567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:32:06.694536516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:32:11.452910369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:56832"
level=debug ts=2026-05-15T12:32:23.76295216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:32:36.696249409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:33:06.697682258Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:33:07.808309684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:36746"
level=debug ts=2026-05-15T12:33:17.932335894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:33:36.699011772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
level=debug ts=2026-05-15T12:34:06.326111209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-15T12:34:06.327783425Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:34:06.700683409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:34:36.702462812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:34:48.226177593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56404"
level=debug ts=2026-05-15T12:34:55.929378792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:35:06.703483055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:35:07.813087528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:45614"
ts=2026-05-15T12:35:11.462175082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:56138"
ts=2026-05-15T12:35:18.227872606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:56558"
ts=2026-05-15T12:35:36.704636005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:35:41.463226023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:55902"
level=debug ts=2026-05-15T12:35:56.589671209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:36:06.706054698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:36:11.464474405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:53770"
ts=2026-05-15T12:36:36.707098887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
level=debug ts=2026-05-15T12:36:39.2272597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:36:41.4654064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:33454"
level=error ts=2026-05-15T12:36:56.182663402Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:36:56.18462441Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:36:56.186569668Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-15T12:36:56.18659175Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-15T12:37:06.708567083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1fc9deb3 10.129.0.72:7946"
ts=2026-05-15T12:37:11.466506722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:35456"
ts=2026-05-15T12:37:18.233513946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:37592"
level=debug ts=2026-05-15T12:37:31.33698746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-15T12:37:36.709699229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-v2jx2-e5f31737 10.129.0.69:7946"
ts=2026-05-15T12:37:48.235074169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.69:60212"
ts=2026-05-15T12:38:06.710755395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-pl5mf-be0d3082 10.129.0.71:7946"
ts=2026-05-15T12:38:11.469459764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.72:52474"