level=info ts=2026-04-22T18:54:32.236747902Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T18:54:32.236780293Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-22T18:54:32.238787166Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T18:54:32.23917945Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T18:54:32.23927518Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.72 inf=eth0
level=debug ts=2026-04-22T18:54:32.239843498Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T18:54:32.239922738Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.72 inf=eth0
level=info ts=2026-04-22T18:54:32.239995812Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-4525eaea
ts=2026-04-22T18:54:32.240062854Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T18:54:32.240074712Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.72 advertisePort=7946
level=debug ts=2026-04-22T18:54:32.240094838Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.72 advertisePort=7946
level=info ts=2026-04-22T18:54:32.240187568Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-22T18:54:32.246396953Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T18:54:32.246416169Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T18:54:32.246429235Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.243661ms
level=error ts=2026-04-22T18:54:32.246439146Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T18:54:32.246453288Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-22T18:54:32.248820469Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:54:32.248833801Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T18:54:32.254477164Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-22T18:54:32.254528128Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T18:54:32.254536723Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-22T18:54:32.254546028Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T18:54:32.254564539Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T18:54:32.254591291Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T18:54:32.254772826Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2026-04-22T18:54:32.254796065Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-22T18:54:32.25480312Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-22T18:54:32.254809091Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-22T18:54:32.254874453Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-22T18:54:32.25491017Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-22T18:54:32.2549151Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T18:54:32.255099025Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-22T18:54:32.255109053Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-22T18:54:32.255124551Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-22T18:54:33.255435696Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-22T18:54:33.433723143Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-22T18:54:33.433777764Z caller=loki.go:599 msg="Loki started" startup_time=1.254896536s
ts=2026-04-22T18:54:33.830302765Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:54:33.830328715Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T18:54:37.224629704Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:54:37.224653708Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T18:54:38.434751827Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-22T18:54:38.434801375Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-22T18:54:41.251331253Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:54:41.251370384Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T18:54:50.91082429Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T18:54:50.910849904Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T18:54:51.228727367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T18:55:07.232107465Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T18:55:14.231754967Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
ts=2026-04-22T18:55:14.231881822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-04-22T18:55:14.231911966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:52526"
ts=2026-04-22T18:55:14.234224658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.78:7946"
ts=2026-04-22T18:55:14.236866275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.81:7946"
ts=2026-04-22T18:55:14.239817941Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.83:7946: dial tcp 10.129.0.83:7946: connect: connection refused"
ts=2026-04-22T18:55:14.241229664Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.79:7946: dial tcp 10.129.0.79:7946: connect: connection refused"
level=info ts=2026-04-22T18:55:14.241238585Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=41.994779057s
ts=2026-04-22T18:55:15.725642918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60022"
ts=2026-04-22T18:55:20.485530432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:54892"
ts=2026-04-22T18:55:22.342937981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53118"
level=debug ts=2026-04-22T18:55:27.536286497Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T18:55:40.0424019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T18:55:54.599819618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T18:56:02.091076744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:39196"
ts=2026-04-22T18:56:07.296202371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50720"
ts=2026-04-22T18:56:10.045580179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
ts=2026-04-22T18:56:20.72441091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35638"
level=debug ts=2026-04-22T18:56:27.603882447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T18:56:27.605564312Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T18:56:40.049069781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T18:57:07.543219735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T18:57:10.052431416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T18:57:40.05552004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T18:58:08.415706028Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T18:58:10.05868179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T18:58:20.732595983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:56254"
ts=2026-04-22T18:58:40.062636089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
ts=2026-04-22T18:59:10.066442051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T18:59:15.471476217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T18:59:40.069393518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
ts=2026-04-22T18:59:50.738487332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:46986"
ts=2026-04-22T19:00:02.109066168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55180"
ts=2026-04-22T19:00:07.30820587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:47066"
ts=2026-04-22T19:00:10.072272689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:00:22.874974166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T19:00:37.311466659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:56374"
ts=2026-04-22T19:00:40.07536559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:01:02.113617023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38872"
ts=2026-04-22T19:01:10.078780509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T19:01:29.532123434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T19:01:29.533993113Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:01:37.316029712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60386"
ts=2026-04-22T19:01:40.082358573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:02:02.11879604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37436"
ts=2026-04-22T19:02:10.085889072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:02:10.335162898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:02:40.088736435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:02:50.749377619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:45842"
level=debug ts=2026-04-22T19:02:57.198213295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:03:10.092051482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:03:37.322263625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44010"
ts=2026-04-22T19:03:40.095715864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:03:59.136581232Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:04:02.125859059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54256"
ts=2026-04-22T19:04:10.098366014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
ts=2026-04-22T19:04:37.32706455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57308"
level=info ts=2026-04-22T19:04:38.434810153Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-22T19:04:38.438392084Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:04:38.440180086Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:04:38.441944993Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:04:38.441972258Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T19:04:38.441984246Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-22T19:04:40.101554998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T19:04:47.561445703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:05:02.130873673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43416"
ts=2026-04-22T19:05:10.104318053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:05:20.756937917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:55176"
level=debug ts=2026-04-22T19:05:30.79748486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:05:30.799468973Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:05:40.10786053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:06:07.333099739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53956"
ts=2026-04-22T19:06:10.111746973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:06:12.8078073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:06:32.138025323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34556"
ts=2026-04-22T19:06:37.336580646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:52416"
ts=2026-04-22T19:06:40.115272933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
ts=2026-04-22T19:07:02.141710598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44274"
level=debug ts=2026-04-22T19:07:08.586642832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:07:10.118624221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:07:40.121533079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:07:50.765534457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35954"
ts=2026-04-22T19:08:10.125035966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T19:08:10.442134221Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:08:40.127975129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:09:07.520923121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:09:10.130813998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:09:32.150509296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41380"
ts=2026-04-22T19:09:40.134228812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:09:50.772207313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:55350"
level=debug ts=2026-04-22T19:10:03.475188541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:10:03.477166797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:10:10.137289044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:10:40.140897734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:11:01.874944525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:11:10.143829719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:11:37.351882735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36006"
ts=2026-04-22T19:11:40.146977049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
ts=2026-04-22T19:11:50.780304075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:60800"
level=debug ts=2026-04-22T19:11:58.202886507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:12:07.356670431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39296"
ts=2026-04-22T19:12:10.150103022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:12:20.783616267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:34142"
ts=2026-04-22T19:12:37.359846246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59832"
ts=2026-04-22T19:12:40.153052552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T19:12:47.070369252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:13:02.164677839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36170"
ts=2026-04-22T19:13:07.363164497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51500"
ts=2026-04-22T19:13:10.156457321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:13:20.788854602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37204"
ts=2026-04-22T19:13:32.168480239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60860"
ts=2026-04-22T19:13:37.366543578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49916"
ts=2026-04-22T19:13:40.159900162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:13:47.527861236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:14:02.171468816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34552"
ts=2026-04-22T19:14:10.163480042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=error ts=2026-04-22T19:14:38.446716685Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:14:38.448705684Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:14:38.450534499Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:14:38.450557215Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:14:40.166939872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:14:47.623546967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:14:47.625146678Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:15:10.170512672Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T19:15:28.367368604Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:15:40.173675332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
ts=2026-04-22T19:16:10.177176184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T19:16:16.970231224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:16:40.18067036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:16:50.808907289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:60334"
ts=2026-04-22T19:17:02.182596918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57234"
level=debug ts=2026-04-22T19:17:07.922143689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:17:10.18399319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:17:37.381373542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58200"
ts=2026-04-22T19:17:40.18711973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
ts=2026-04-22T19:17:50.814148886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:58066"
level=debug ts=2026-04-22T19:18:03.954463125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:18:10.191104019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:18:40.194754367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:18:50.819045164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40588"
level=debug ts=2026-04-22T19:19:04.510058786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:19:04.511676765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:19:10.198397691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:19:20.822369973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:50758"
ts=2026-04-22T19:19:32.196408524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47706"
ts=2026-04-22T19:19:40.201757198Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T19:19:42.705615496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:20:10.204899077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:20:30.766045692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:20:40.208673828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:21:02.204792692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:54840"
level=debug ts=2026-04-22T19:21:06.230308975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:21:10.21182868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:21:40.214775804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-58a93aa4 10.129.0.82:7946"
level=debug ts=2026-04-22T19:21:52.131442212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:22:10.217732233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:22:15.524138208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40834"
ts=2026-04-22T19:22:15.532952577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40836"
ts=2026-04-22T19:22:24.24107431Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-58a93aa4 (timeout reached)"
ts=2026-04-22T19:22:27.241405177Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-58a93aa4 has failed, no acks received"
ts=2026-04-22T19:22:32.210971454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38580"
ts=2026-04-22T19:22:40.221389728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-755d070c 10.129.0.82:7946"
ts=2026-04-22T19:22:42.716995607Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-58a93aa4 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-22T19:22:52.981060035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:22:52.982705083Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:23:07.403150476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60390"
ts=2026-04-22T19:23:10.225196596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:23:22.241478234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:36616"
ts=2026-04-22T19:23:40.228668528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:23:50.238500374Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:24:10.232176517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-755d070c 10.129.0.82:7946"
ts=2026-04-22T19:24:37.409583208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51326"
level=debug ts=2026-04-22T19:24:37.906547768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:24:38.44510425Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:24:38.447287494Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:24:38.44935514Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:24:38.449381504Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:24:40.239511265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:25:10.246800078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:25:27.086302629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:25:37.415862676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38296"
ts=2026-04-22T19:25:40.250047464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:25:52.249058918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:33806"
ts=2026-04-22T19:26:07.419190882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34560"
ts=2026-04-22T19:26:10.253566265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:26:22.253477087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35156"
level=debug ts=2026-04-22T19:26:26.366012759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:26:40.257848117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:26:52.256584461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:54156"
ts=2026-04-22T19:27:02.225729157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40576"
ts=2026-04-22T19:27:10.261539156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:27:24.346790358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:27:24.349120916Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:27:40.265110422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-755d070c 10.129.0.82:7946"
ts=2026-04-22T19:28:02.230575966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40230"
level=debug ts=2026-04-22T19:28:07.47045854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:28:10.268264023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:28:32.233601282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36822"
ts=2026-04-22T19:28:40.271964147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-755d070c 10.129.0.82:7946"
level=debug ts=2026-04-22T19:28:53.523815257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:29:02.23702343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48830"
ts=2026-04-22T19:29:07.432149602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:57492"
ts=2026-04-22T19:29:10.275112836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-755d070c 10.129.0.82:7946"
level=debug ts=2026-04-22T19:29:35.323715167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:29:37.435771462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49166"
ts=2026-04-22T19:29:40.277890665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:29:52.265987196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:52520"
ts=2026-04-22T19:30:10.280673006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:30:11.330113301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:30:29.240413747Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-755d070c (timeout reached)"
ts=2026-04-22T19:30:32.240416618Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-755d070c has failed, no acks received"
ts=2026-04-22T19:30:32.242879164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41232"
ts=2026-04-22T19:30:37.440318553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35066"
ts=2026-04-22T19:30:40.284051789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:30:49.240904998Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-755d070c (timeout reached)"
ts=2026-04-22T19:30:52.241540592Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-755d070c has failed, no acks received"
ts=2026-04-22T19:30:52.241663921Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-755d070c as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:30:52.288558943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:46502"
ts=2026-04-22T19:30:52.297645158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:46512"
ts=2026-04-22T19:31:10.287120727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:31:13.586942445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:31:13.588394968Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:31:40.290346393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:32:02.248649591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47452"
level=debug ts=2026-04-22T19:32:09.11905341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:32:10.293491998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:32:32.251646657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47284"
ts=2026-04-22T19:32:40.29670028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:33:02.255043047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:32982"
level=debug ts=2026-04-22T19:33:04.382137619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:33:07.448099423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37466"
ts=2026-04-22T19:33:10.299882537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-89586d17 10.129.0.82:7946"
ts=2026-04-22T19:33:14.426602826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40530"
ts=2026-04-22T19:33:40.303445196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:33:44.430491825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:47442"
level=debug ts=2026-04-22T19:33:52.268077591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:34:10.306855203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:34:37.454301881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45238"
level=error ts=2026-04-22T19:34:38.446928355Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:34:38.44983161Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:34:38.452321359Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:34:38.452392373Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:34:40.310298449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:34:51.076596043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:35:10.313733591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:35:14.437543907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:53632"
level=debug ts=2026-04-22T19:35:37.96649633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:35:37.968214838Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:35:40.317139782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:36:10.320429314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:36:36.798747337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:36:37.461520844Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36454"
ts=2026-04-22T19:36:40.323817388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:36:44.443105208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:46186"
ts=2026-04-22T19:37:07.464817978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:54494"
ts=2026-04-22T19:37:10.326904928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:37:18.706986551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:37:32.270472228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59770"
ts=2026-04-22T19:37:40.33019535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:37:44.44725295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:47192"
ts=2026-04-22T19:38:10.333633261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-89586d17 10.129.0.82:7946"
level=debug ts=2026-04-22T19:38:20.698416547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:38:37.470034328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:46782"
ts=2026-04-22T19:38:40.337176266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-89586d17 10.129.0.82:7946"
ts=2026-04-22T19:38:44.451054258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:45218"
ts=2026-04-22T19:39:02.276817738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49842"
ts=2026-04-22T19:39:07.474006423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58394"
ts=2026-04-22T19:39:10.340733035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:39:16.093591843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:39:37.476874525Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48934"
ts=2026-04-22T19:39:40.343907607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-89586d17 10.129.0.82:7946"
ts=2026-04-22T19:39:44.454925415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:47446"
ts=2026-04-22T19:40:10.347895871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:40:14.458526766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:51742"
level=debug ts=2026-04-22T19:40:18.141714331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:40:18.144009819Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:40:40.350874766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-89586d17 10.129.0.82:7946"
ts=2026-04-22T19:40:44.462135931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:33156"
level=debug ts=2026-04-22T19:40:59.179817719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:41:07.483372042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39564"
ts=2026-04-22T19:41:10.354097483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:41:37.48658939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59100"
ts=2026-04-22T19:41:40.357553943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:41:47.017173283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:42:07.489752569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37296"
ts=2026-04-22T19:42:10.360365877Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-89586d17 10.129.0.82:7946"
ts=2026-04-22T19:42:14.468422076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:49624"
ts=2026-04-22T19:42:30.222187971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:57386"
ts=2026-04-22T19:42:30.231260687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:57388"
ts=2026-04-22T19:42:32.288840459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60760"
level=debug ts=2026-04-22T19:42:34.194475585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:42:34.241737086Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-89586d17 (timeout reached)"
ts=2026-04-22T19:42:37.241639678Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-89586d17 has failed, no acks received"
ts=2026-04-22T19:42:39.242437534Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-89586d17 (timeout reached)"
ts=2026-04-22T19:42:40.363817762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:42:42.242210824Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-89586d17 has failed, no acks received"
ts=2026-04-22T19:42:57.242090904Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-89586d17 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:43:02.292057783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37682"
ts=2026-04-22T19:43:07.494450737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51236"
ts=2026-04-22T19:43:10.367089733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:43:20.604108966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:43:24.616935571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:36030"
ts=2026-04-22T19:43:37.49768618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43906"
ts=2026-04-22T19:43:40.370203063Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:43:54.620213687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:37244"
ts=2026-04-22T19:44:02.296516616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36982"
ts=2026-04-22T19:44:10.373967103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:44:21.709379915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:44:21.71147009Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:44:24.623976683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:45690"
level=error ts=2026-04-22T19:44:38.446020632Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:44:38.447864565Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:44:38.449613427Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:44:38.449637537Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:44:40.376727862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:45:04.929067534Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:45:07.505138673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:43594"
ts=2026-04-22T19:45:10.379739677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1e4f1756 10.129.0.82:7946"
ts=2026-04-22T19:45:24.627982611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:42312"
ts=2026-04-22T19:45:40.383242639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:46:05.241931867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:46:10.386160157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1e4f1756 10.129.0.82:7946"
ts=2026-04-22T19:46:24.632647654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:57902"
ts=2026-04-22T19:46:32.305654248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:32892"
ts=2026-04-22T19:46:40.389176686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1e4f1756 10.129.0.82:7946"
level=debug ts=2026-04-22T19:46:53.672003992Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:46:54.635729632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:42610"
ts=2026-04-22T19:47:10.392093458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:47:40.3957307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:47:44.865701887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:48:10.398995669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:48:24.641873912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35246"
ts=2026-04-22T19:48:40.402412168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:48:42.890231642Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:48:42.891932462Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:49:10.405329518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:49:24.646209795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:52432"
ts=2026-04-22T19:49:32.315023428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60038"
ts=2026-04-22T19:49:37.519109357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:44016"
level=debug ts=2026-04-22T19:49:39.762563612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:49:40.408413177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:49:54.649255218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:58450"
ts=2026-04-22T19:50:10.412040476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1e4f1756 10.129.0.82:7946"
level=debug ts=2026-04-22T19:50:38.733732981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:50:40.415433654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:50:54.654668155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:46662"
ts=2026-04-22T19:51:07.525475576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37444"
ts=2026-04-22T19:51:10.418562177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:51:24.657862228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:48162"
level=debug ts=2026-04-22T19:51:27.861185665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:51:32.322131837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56958"
ts=2026-04-22T19:51:40.423391357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:52:02.32556408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41988"
level=debug ts=2026-04-22T19:52:02.382657622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:52:07.530028751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39840"
ts=2026-04-22T19:52:10.426405845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:52:32.328613588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46124"
level=debug ts=2026-04-22T19:52:39.301264691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:52:39.30292928Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:52:40.42939122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1e4f1756 10.129.0.82:7946"
ts=2026-04-22T19:53:10.432478823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:53:24.665241765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:54658"
ts=2026-04-22T19:53:32.333804798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41640"
ts=2026-04-22T19:53:37.53571381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:34732"
ts=2026-04-22T19:53:40.4362156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1e4f1756 10.129.0.82:7946"
level=debug ts=2026-04-22T19:53:41.331414117Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:54:07.538928534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:37284"
ts=2026-04-22T19:54:10.440128382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1e4f1756 10.129.0.82:7946"
level=debug ts=2026-04-22T19:54:19.990529159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:54:32.338557703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48810"
level=error ts=2026-04-22T19:54:38.446582951Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:54:38.448388865Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:54:38.450322795Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T19:54:38.450368075Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:54:40.443136955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:54:54.671187652Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:59876"
ts=2026-04-22T19:55:02.341955682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33150"
level=debug ts=2026-04-22T19:55:06.801376136Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:55:10.446784969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:55:11.76035511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:56806"
ts=2026-04-22T19:55:11.769057924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:56822"
ts=2026-04-22T19:55:14.241952873Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1e4f1756 (timeout reached)"
ts=2026-04-22T19:55:17.24136271Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1e4f1756 has failed, no acks received"
ts=2026-04-22T19:55:34.241205752Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1e4f1756 (timeout reached)"
ts=2026-04-22T19:55:37.241948949Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-1e4f1756 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T19:55:37.242194451Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1e4f1756 has failed, no acks received"
ts=2026-04-22T19:55:40.44956369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
level=debug ts=2026-04-22T19:55:54.263316528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:56:07.546135872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:38116"
ts=2026-04-22T19:56:10.452846619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T19:56:32.347653125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51206"
level=debug ts=2026-04-22T19:56:38.592431384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T19:56:38.594122052Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T19:56:40.456090859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:57:07.552026958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:58168"
ts=2026-04-22T19:57:10.459650298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:57:22.757446739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:57:40.463200478Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
level=debug ts=2026-04-22T19:58:07.050614088Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:58:07.556075834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50328"
ts=2026-04-22T19:58:10.466760457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T19:58:14.069682029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:42686"
ts=2026-04-22T19:58:32.354221403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43428"
ts=2026-04-22T19:58:40.47017143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T19:58:54.779068334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T19:59:10.473809682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
ts=2026-04-22T19:59:14.07476958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:53574"
ts=2026-04-22T19:59:40.477415571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T19:59:55.605713125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:00:02.359207034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42566"
ts=2026-04-22T20:00:07.561303059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:51762"
ts=2026-04-22T20:00:10.481145239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:00:40.484214629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
level=debug ts=2026-04-22T20:00:49.910653765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:00:49.912384523Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:01:10.487156291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:01:37.566885287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42566"
level=debug ts=2026-04-22T20:01:39.245652196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:01:40.490310172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:02:02.366212489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38058"
ts=2026-04-22T20:02:10.493394159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
ts=2026-04-22T20:02:14.083213405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:59938"
level=debug ts=2026-04-22T20:02:25.666062334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:02:32.369500567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36412"
ts=2026-04-22T20:02:40.496865482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
ts=2026-04-22T20:02:44.086824401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:38012"
ts=2026-04-22T20:03:07.573040947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:49402"
ts=2026-04-22T20:03:10.500143787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
level=debug ts=2026-04-22T20:03:21.343221386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:03:40.503593678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
ts=2026-04-22T20:03:44.091295714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:44218"
level=debug ts=2026-04-22T20:03:55.338122803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:04:07.577825484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60004"
ts=2026-04-22T20:04:10.507561359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
ts=2026-04-22T20:04:14.094007596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:38690"
level=error ts=2026-04-22T20:04:38.44471213Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:04:38.446409559Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:04:38.447998191Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:04:38.448018858Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T20:04:38.599359396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:04:38.60093865Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:04:40.510586517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:05:07.582949195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:41344"
ts=2026-04-22T20:05:10.514017726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:05:20.85394129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:05:40.517171605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:05:44.100206979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:33310"
ts=2026-04-22T20:06:10.520895515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
level=debug ts=2026-04-22T20:06:13.723267051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:06:40.523843466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:07:10.527257421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:07:11.396248972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:07:40.53083491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
ts=2026-04-22T20:07:44.108475696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:38960"
level=debug ts=2026-04-22T20:08:02.185756112Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:08:10.533637528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T20:08:14.111768754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40208"
ts=2026-04-22T20:08:40.537373979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-948ee418 10.129.0.82:7946"
level=debug ts=2026-04-22T20:08:43.294088435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:08:43.295815262Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:09:10.54072603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:09:14.115813256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:42564"
ts=2026-04-22T20:09:40.543870297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:09:43.896439901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:10:07.599355929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39376"
ts=2026-04-22T20:10:10.547137798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:10:14.123954986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:52732"
level=debug ts=2026-04-22T20:10:33.199673968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:10:40.550416289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:10:44.127478911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:34700"
ts=2026-04-22T20:10:59.027274975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:43282"
ts=2026-04-22T20:10:59.035441618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:43286"
ts=2026-04-22T20:11:09.241855566Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-948ee418 (timeout reached)"
ts=2026-04-22T20:11:10.553308401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T20:11:12.202459897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:11:12.241726691Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-948ee418 has failed, no acks received"
ts=2026-04-22T20:11:14.241845739Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-948ee418 (timeout reached)"
ts=2026-04-22T20:11:17.24207707Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-948ee418 has failed, no acks received"
ts=2026-04-22T20:11:32.242741545Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-948ee418 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:11:34.240742927Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-948ee418 (timeout reached)"
ts=2026-04-22T20:11:37.240811168Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-948ee418 has failed, no acks received"
ts=2026-04-22T20:11:40.556775944Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b0748a9 10.129.0.82:7946"
ts=2026-04-22T20:11:42.732965851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:34912"
level=debug ts=2026-04-22T20:11:50.710799908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:12:02.396577324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44246"
ts=2026-04-22T20:12:07.605444406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:35844"
ts=2026-04-22T20:12:10.559525442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:12:32.400380388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45646"
ts=2026-04-22T20:12:40.562281584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:12:42.737861181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:36914"
level=debug ts=2026-04-22T20:12:47.507258102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:12:47.508942484Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:13:10.565874298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T20:13:32.405262193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53216"
ts=2026-04-22T20:13:40.569378768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:13:40.638533845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:13:42.742221362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:56214"
ts=2026-04-22T20:14:07.612130184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42794"
ts=2026-04-22T20:14:10.57265721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T20:14:20.59738493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:14:37.615175618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:39388"
level=error ts=2026-04-22T20:14:38.446054554Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:14:38.448065743Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:14:38.454634478Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:14:38.454661924Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:14:40.575825368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:15:10.579518288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:15:14.092630219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:15:40.582517441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b0748a9 10.129.0.82:7946"
ts=2026-04-22T20:15:42.749111383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:60868"
ts=2026-04-22T20:16:02.413890539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36248"
ts=2026-04-22T20:16:10.585985434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:16:12.75177405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40792"
level=debug ts=2026-04-22T20:16:14.252426488Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:16:32.41702657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53162"
ts=2026-04-22T20:16:40.589641378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b0748a9 10.129.0.82:7946"
level=debug ts=2026-04-22T20:16:48.631351273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:16:48.633278546Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:17:02.42013283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56824"
ts=2026-04-22T20:17:07.623883642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:60990"
ts=2026-04-22T20:17:10.593040496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b0748a9 10.129.0.82:7946"
ts=2026-04-22T20:17:40.596601508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:17:47.988491244Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:18:10.599502633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:18:12.759470101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:46200"
level=debug ts=2026-04-22T20:18:24.620926178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:18:40.603537962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:18:42.762526407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:35818"
ts=2026-04-22T20:19:10.606231909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b0748a9 10.129.0.82:7946"
level=debug ts=2026-04-22T20:19:14.21691271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:19:19.241227875Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7b0748a9 (timeout reached)"
ts=2026-04-22T20:19:22.24213218Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7b0748a9 has failed, no acks received"
ts=2026-04-22T20:19:24.243222781Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7b0748a9 (timeout reached)"
ts=2026-04-22T20:19:27.242530875Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7b0748a9 has failed, no acks received"
ts=2026-04-22T20:19:30.290741164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:38886"
ts=2026-04-22T20:19:30.300138002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:38894"
ts=2026-04-22T20:19:40.609463529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:19:42.242219634Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-7b0748a9 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-22T20:19:44.241040494Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7b0748a9 (timeout reached)"
ts=2026-04-22T20:19:47.241016866Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7b0748a9 has failed, no acks received"
ts=2026-04-22T20:20:07.633551464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50046"
ts=2026-04-22T20:20:10.612167705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:20:15.046929022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:20:40.615458605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:21:07.638177581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:59536"
ts=2026-04-22T20:21:10.618584781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
level=debug ts=2026-04-22T20:21:12.550553785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:21:12.552187822Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:21:37.140778958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:55078"
ts=2026-04-22T20:21:40.621660555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
level=debug ts=2026-04-22T20:22:03.327871213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:22:07.144529581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:41446"
ts=2026-04-22T20:22:10.625084939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:22:40.62862987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:23:00.819706599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:23:10.63204622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
ts=2026-04-22T20:23:40.63525928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
level=debug ts=2026-04-22T20:23:41.049639863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:24:07.151074919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:40386"
ts=2026-04-22T20:24:10.638998745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:24:32.44126053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52160"
ts=2026-04-22T20:24:37.154527233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:56924"
level=debug ts=2026-04-22T20:24:37.404475653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:24:37.648633527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:36018"
level=error ts=2026-04-22T20:24:38.444757621Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:24:38.44645959Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:24:38.448027734Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-22T20:24:38.448051169Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:24:40.641856601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
ts=2026-04-22T20:25:07.157711538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:36552"
ts=2026-04-22T20:25:07.651852646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42404"
ts=2026-04-22T20:25:10.64499616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
level=debug ts=2026-04-22T20:25:18.566462366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:25:18.568072873Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:25:32.444800338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37392"
ts=2026-04-22T20:25:37.161375995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:41776"
ts=2026-04-22T20:25:37.654813741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:42434"
ts=2026-04-22T20:25:40.648105216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:25:57.946050014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:26:02.447920045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33906"
ts=2026-04-22T20:26:10.651847654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
ts=2026-04-22T20:26:37.165644129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:54038"
ts=2026-04-22T20:26:40.655196656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T20:26:56.307429843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:27:10.658467932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
ts=2026-04-22T20:27:32.453744671Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34292"
ts=2026-04-22T20:27:40.661532687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T20:27:55.85492085Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:28:10.664847665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
ts=2026-04-22T20:28:32.457559972Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53216"
ts=2026-04-22T20:28:40.668166567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
level=debug ts=2026-04-22T20:28:45.944373648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:29:10.671375382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T20:29:22.918776129Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T20:29:22.920663326Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-22T20:29:37.668377238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:45896"
ts=2026-04-22T20:29:40.674572313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=debug ts=2026-04-22T20:30:01.103453275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:30:10.677881001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
ts=2026-04-22T20:30:32.463880745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51790"
ts=2026-04-22T20:30:37.673722047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:53650"
ts=2026-04-22T20:30:40.681024742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:30:45.039417692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:31:02.467167036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58354"
ts=2026-04-22T20:31:07.178064865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:45516"
ts=2026-04-22T20:31:07.676907817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:48668"
ts=2026-04-22T20:31:10.683851389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-ld99g-e70e3abe 10.129.0.81:7946"
level=debug ts=2026-04-22T20:31:38.418622864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:31:40.687153586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6918225f 10.129.0.82:7946"
ts=2026-04-22T20:31:54.413911919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:41906"
ts=2026-04-22T20:31:54.422292568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.82:41920"
ts=2026-04-22T20:32:10.689888954Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7174df1 10.129.0.82:7946"
ts=2026-04-22T20:32:14.241005981Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6918225f (timeout reached)"
ts=2026-04-22T20:32:17.240962163Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6918225f has failed, no acks received"
level=debug ts=2026-04-22T20:32:34.993243228Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T20:32:40.693455034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-c7174df1 10.129.0.82:7946"