level=info ts=2026-04-20T19:04:42.337185168Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T19:04:42.337221243Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T19:04:42.338933397Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T19:04:42.339349455Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T19:04:42.339453229Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.65 inf=eth0
level=debug ts=2026-04-20T19:04:42.340050309Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T19:04:42.340135867Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.65 inf=eth0
level=info ts=2026-04-20T19:04:42.340221335Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-eb93afb5
ts=2026-04-20T19:04:42.340304711Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T19:04:42.340320206Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.65 advertisePort=7946
level=debug ts=2026-04-20T19:04:42.340339652Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.65 advertisePort=7946
level=info ts=2026-04-20T19:04:42.340494288Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T19:04:42.351098755Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T19:04:42.351123202Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T19:04:42.351136843Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=10.646001ms
level=error ts=2026-04-20T19:04:42.351148476Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T19:04:42.35116326Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T19:04:42.35477652Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T19:04:42.354799439Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T19:04:42.357150883Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-20T19:04:42.357508002Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-20T19:04:42.357533598Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-20T19:04:42.357568803Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-20T19:04:42.357579826Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-20T19:04:42.357892242Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-20T19:04:42.357905396Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-20T19:04:42.357988622Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-20T19:04:42.357995395Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-20T19:04:42.358000635Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T19:04:42.358048909Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T19:04:42.358091383Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T19:04:42.358110745Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T19:04:42.358531969Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T19:04:42.358547019Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T19:04:42.358573356Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T19:04:43.359115619Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T19:04:43.459624727Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T19:04:43.459691619Z caller=loki.go:599 msg="Loki started" startup_time=1.182892971s
ts=2026-04-20T19:04:43.936274402Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T19:04:43.936300878Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:04:46.370157008Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T19:04:46.370183023Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T19:04:48.460256693Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T19:04:48.460323277Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
level=debug ts=2026-04-20T19:04:51.216709525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:04:53.662943451Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T19:04:53.662970144Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:05:00.229019597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:36372"
ts=2026-04-20T19:05:05.37905864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:50812"
ts=2026-04-20T19:05:06.978507612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:57184"
ts=2026-04-20T19:05:08.481351531Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.78:7946: dial tcp 10.129.0.78:7946: connect: connection refused"
ts=2026-04-20T19:05:08.481553299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.64:7946"
ts=2026-04-20T19:05:08.482230674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.65:7946"
ts=2026-04-20T19:05:08.482247399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.65:59524"
ts=2026-04-20T19:05:08.484453665Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.92:7946: dial tcp 10.128.0.92:7946: connect: connection refused"
level=info ts=2026-04-20T19:05:08.484466239Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=26.13329619s
level=debug ts=2026-04-20T19:05:12.631408768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:05:13.51204993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:05:32.2785798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:05:37.858979182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55024"
ts=2026-04-20T19:05:43.513253873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:05:57.572281961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:05:59.346492012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:56440"
ts=2026-04-20T19:06:07.860815175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33136"
ts=2026-04-20T19:06:08.944558827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35660"
ts=2026-04-20T19:06:13.51440879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:06:29.349486682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:33810"
level=debug ts=2026-04-20T19:06:30.236975854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T19:06:30.238629798Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:06:37.861680899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42324"
ts=2026-04-20T19:06:43.515389142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:07:08.529775123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:07:08.949101183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36010"
ts=2026-04-20T19:07:13.51628707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:07:29.355489734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:51806"
ts=2026-04-20T19:07:43.517378207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:08:13.519547233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:08:19.029381298Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:08:43.521634535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:08:59.365995795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:37978"
ts=2026-04-20T19:09:13.523364956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:09:34.276917961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:09:43.524418784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:09:59.372267165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:35072"
ts=2026-04-20T19:10:13.526030681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:10:29.375335366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44656"
level=debug ts=2026-04-20T19:10:36.695972587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T19:10:43.527397586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:11:07.882829802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44622"
ts=2026-04-20T19:11:08.967312685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:46398"
ts=2026-04-20T19:11:13.529988291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:11:27.61764616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T19:11:27.619232083Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:11:29.381930832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:54838"
ts=2026-04-20T19:11:38.968097978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35934"
ts=2026-04-20T19:11:43.532691348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:11:59.385196298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:38886"
ts=2026-04-20T19:12:08.968958531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60586"
ts=2026-04-20T19:12:13.53568165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:12:27.407490769Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:12:29.38857042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:56368"
ts=2026-04-20T19:12:38.969927063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:33342"
ts=2026-04-20T19:12:43.538464374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:12:59.3920546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:54528"
ts=2026-04-20T19:13:07.889039829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44634"
ts=2026-04-20T19:13:08.971162083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:45112"
ts=2026-04-20T19:13:13.539864694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:13:13.970356367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:13:43.542577442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:14:11.260382712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:14:13.545615457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:14:29.401473923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:51500"
ts=2026-04-20T19:14:38.979182623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43574"
ts=2026-04-20T19:14:43.547660198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=info ts=2026-04-20T19:14:48.460999737Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T19:14:48.466483943Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:14:48.468506695Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:14:48.470169735Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:14:48.470199875Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T19:14:48.470211772Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-20T19:14:54.817852428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:14:59.404482934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:35794"
ts=2026-04-20T19:15:13.548978922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:15:29.407270526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:44454"
ts=2026-04-20T19:15:37.901142289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:58020"
ts=2026-04-20T19:15:43.550350389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:15:56.091920159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:15:56.093638839Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:16:07.902527731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52232"
ts=2026-04-20T19:16:13.551936018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:16:29.413691119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:54014"
ts=2026-04-20T19:16:37.904059908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47870"
level=debug ts=2026-04-20T19:16:43.156157551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:16:43.553474609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:17:07.905772643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50742"
ts=2026-04-20T19:17:08.991910034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:50960"
ts=2026-04-20T19:17:13.554972256Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:17:23.434679259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:17:37.907132032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:41938"
ts=2026-04-20T19:17:38.993469116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35972"
ts=2026-04-20T19:17:43.556357019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:18:08.994892603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34776"
ts=2026-04-20T19:18:13.558934292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:18:14.33634975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:18:43.56125818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:18:51.360976865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:19:13.562999698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:19:31.688524378Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:19:31.690106107Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:19:43.564602071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:20:13.566414187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:20:29.439822211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:60208"
level=debug ts=2026-04-20T19:20:31.072220381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:20:43.567550119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:20:59.443351695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:58666"
ts=2026-04-20T19:21:13.569370744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:21:26.837039742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:21:43.570815499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:22:07.924114094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:48036"
ts=2026-04-20T19:22:09.00918933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38276"
level=debug ts=2026-04-20T19:22:09.459463855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:22:13.57360633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:22:29.455902966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:41760"
ts=2026-04-20T19:22:37.925635232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39846"
ts=2026-04-20T19:22:39.01021762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:57510"
ts=2026-04-20T19:22:43.57528362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:22:59.459536077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:38300"
level=debug ts=2026-04-20T19:23:06.216385341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:23:09.012017073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38602"
ts=2026-04-20T19:23:13.577987774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:23:39.014011198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:32946"
level=debug ts=2026-04-20T19:23:40.884060496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:23:40.885710041Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:23:43.579857611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:24:13.581473206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:24:38.915271851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:24:43.584241779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=error ts=2026-04-20T19:24:48.473698698Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:24:48.475812523Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:24:48.477828911Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:24:48.477861744Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:25:09.022567259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54132"
ts=2026-04-20T19:25:13.587364218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:25:29.476141367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:53664"
level=debug ts=2026-04-20T19:25:40.559696164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:25:43.590027804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:25:59.479652235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:50814"
ts=2026-04-20T19:26:13.591605746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:26:34.68540203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:26:37.944673441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:52948"
ts=2026-04-20T19:26:43.594410217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:27:07.945390641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38880"
ts=2026-04-20T19:27:09.029894693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37352"
ts=2026-04-20T19:27:13.596766872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:27:23.067393214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:27:37.946298517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47584"
ts=2026-04-20T19:27:43.598498538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:28:13.600246093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:28:19.883181236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:28:19.885206717Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:28:37.948127272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33108"
ts=2026-04-20T19:28:43.601372157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:29:07.949655117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59758"
ts=2026-04-20T19:29:13.603545961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:29:20.621791756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:29:37.950524779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:49222"
ts=2026-04-20T19:29:43.60649798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:30:07.951371265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:35782"
ts=2026-04-20T19:30:13.609203966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:30:19.245048412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:30:29.510694807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:59256"
ts=2026-04-20T19:30:43.611868747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:31:08.951874303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:31:13.615096301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:31:39.047127909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:50476"
level=debug ts=2026-04-20T19:31:43.418292425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:31:43.618197792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:32:09.048575463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53982"
ts=2026-04-20T19:32:13.620378962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:32:29.522573316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:40470"
ts=2026-04-20T19:32:39.050401101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48290"
level=debug ts=2026-04-20T19:32:43.102855736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:32:43.104625979Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:32:43.621923619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:32:59.526162901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:35072"
ts=2026-04-20T19:33:07.967665802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:33928"
ts=2026-04-20T19:33:09.051834464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35182"
ts=2026-04-20T19:33:13.624768917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:33:32.721096315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:33:39.053789388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:54228"
ts=2026-04-20T19:33:43.62670088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:34:09.055054922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48100"
level=debug ts=2026-04-20T19:34:13.157490242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:34:13.628116268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:34:37.973752755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:39416"
ts=2026-04-20T19:34:39.05684622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:32778"
ts=2026-04-20T19:34:43.629268553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=error ts=2026-04-20T19:34:48.474571083Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:34:48.476398901Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:34:48.478053997Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:34:48.478084681Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T19:34:51.43849119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:34:59.538224036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:39786"
ts=2026-04-20T19:35:09.058504986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35128"
ts=2026-04-20T19:35:13.630403577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:35:29.541537192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:56640"
ts=2026-04-20T19:35:37.977725665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45648"
ts=2026-04-20T19:35:39.060410611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:47482"
level=debug ts=2026-04-20T19:35:40.516379803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:35:43.631277479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:35:59.544692177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:42266"
ts=2026-04-20T19:36:07.978640488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42930"
ts=2026-04-20T19:36:13.632436411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:36:29.547717684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:43570"
level=debug ts=2026-04-20T19:36:35.261669535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:36:35.263601137Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:36:43.63404463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:37:07.983383442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:53846"
ts=2026-04-20T19:37:09.064880304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52582"
ts=2026-04-20T19:37:13.63495472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:37:21.529120467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:37:39.065877032Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:53716"
ts=2026-04-20T19:37:43.63663006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:38:13.637476753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:38:17.46618732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:38:43.638771031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:38:54.04992166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:39:07.988700538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40616"
ts=2026-04-20T19:39:09.072792457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48284"
ts=2026-04-20T19:39:13.640966002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:39:29.568302209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:33656"
ts=2026-04-20T19:39:37.989876269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:55932"
ts=2026-04-20T19:39:43.642892524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:39:46.322595641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:40:09.077422473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:60682"
ts=2026-04-20T19:40:13.644573112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:40:29.574607126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:34894"
level=debug ts=2026-04-20T19:40:32.108406278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:40:32.110285208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:40:37.995320989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40982"
ts=2026-04-20T19:40:39.078767464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:43112"
ts=2026-04-20T19:40:43.645712493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:41:09.080584665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36184"
ts=2026-04-20T19:41:13.647521644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:41:29.581076552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:49584"
level=debug ts=2026-04-20T19:41:34.088016656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:41:43.648719777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:41:59.584368007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:49330"
ts=2026-04-20T19:42:13.650490569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:42:22.162542548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:42:38.001332142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:34904"
ts=2026-04-20T19:42:43.651634608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:42:59.591319985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:55796"
ts=2026-04-20T19:43:13.653227158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:43:23.090176427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:43:29.594691042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:56060"
ts=2026-04-20T19:43:43.654511455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:44:13.656219423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:44:20.087666367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:44:29.601954993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:56292"
ts=2026-04-20T19:44:39.098115715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:37116"
ts=2026-04-20T19:44:43.65811296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=error ts=2026-04-20T19:44:48.475396009Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:44:48.477805044Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:44:48.48002618Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:44:48.480057288Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:44:59.604823858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:35256"
ts=2026-04-20T19:45:09.099477513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36364"
ts=2026-04-20T19:45:13.660673243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:45:14.721943186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:45:14.723761737Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:45:38.011663309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:57512"
ts=2026-04-20T19:45:43.662833849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:45:54.849807432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:46:13.66512513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:46:43.667950428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:46:54.498971829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:47:09.111175943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:52618"
ts=2026-04-20T19:47:13.669873341Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
ts=2026-04-20T19:47:29.621338781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:36516"
ts=2026-04-20T19:47:39.11244062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:35732"
ts=2026-04-20T19:47:43.671214405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:47:48.645028155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:48:08.022412399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:50900"
ts=2026-04-20T19:48:09.113901627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:34224"
ts=2026-04-20T19:48:13.672811747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:48:29.572005447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:48:43.674169681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:48:59.631156073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:56600"
ts=2026-04-20T19:49:08.025186966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:59590"
ts=2026-04-20T19:49:13.675599147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:49:26.646561608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:49:26.648257277Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:49:29.634278236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:47926"
ts=2026-04-20T19:49:43.67899776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:49:59.637945343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:59556"
ts=2026-04-20T19:50:13.682207062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:50:28.909154764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:50:29.641282396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:41598"
ts=2026-04-20T19:50:43.684300189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:51:13.685573795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:51:21.155261912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:51:29.648196426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:45748"
ts=2026-04-20T19:51:39.131293703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:48846"
ts=2026-04-20T19:51:43.688244733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:51:59.303560534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:52:08.037368058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:47408"
ts=2026-04-20T19:52:13.689928353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:52:29.654153791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:59164"
ts=2026-04-20T19:52:38.038506239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:45474"
ts=2026-04-20T19:52:39.133411353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:58258"
level=debug ts=2026-04-20T19:52:42.896602432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:52:43.692560548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:53:13.694983542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:53:21.94525713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:53:21.946827468Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:53:43.696251757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:54:09.137460467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:44570"
ts=2026-04-20T19:54:13.697960233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
level=debug ts=2026-04-20T19:54:20.652638787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:54:43.699530612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=error ts=2026-04-20T19:54:48.47729388Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:54:48.479424208Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:54:48.481422713Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T19:54:48.481460693Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:55:09.142113698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:51166"
ts=2026-04-20T19:55:13.70050868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:55:16.035927309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:55:29.6735982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:38174"
ts=2026-04-20T19:55:38.053576751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:60944"
ts=2026-04-20T19:55:43.703026519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:55:54.598720242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:55:59.676492532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.93:43990"
ts=2026-04-20T19:56:09.147174188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:36168"
ts=2026-04-20T19:56:13.706603496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
ts=2026-04-20T19:56:38.056901973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:44318"
ts=2026-04-20T19:56:43.710055399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:56:44.869961676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:57:08.057831568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:40018"
ts=2026-04-20T19:57:13.71358919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-fc547bc3 10.128.0.93:7946"
level=debug ts=2026-04-20T19:57:31.607729239Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T19:57:31.609357508Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T19:57:38.059667973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:38272"
ts=2026-04-20T19:57:39.152614644Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.64:38090"
ts=2026-04-20T19:57:43.716012524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"
ts=2026-04-20T19:58:08.061265342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.61:42562"
ts=2026-04-20T19:58:13.717092037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xn2l-27c8e3fb 10.130.0.64:7946"
level=debug ts=2026-04-20T19:58:32.831290994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T19:58:43.718587022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ghjsr-e92023b4 10.130.0.61:7946"