level=info ts=2026-04-28T09:57:58.914923792Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-28T09:57:58.914993251Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-28T09:57:58.915947248Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-28T09:57:58.916297335Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T09:57:58.91639617Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.80 inf=eth0
level=debug ts=2026-04-28T09:57:58.917406862Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T09:57:58.917481198Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.80 inf=eth0
level=info ts=2026-04-28T09:57:58.917580023Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-5f8632b4
ts=2026-04-28T09:57:58.917695127Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-28T09:57:58.917710817Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.80 advertisePort=7946
level=debug ts=2026-04-28T09:57:58.917726453Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.80 advertisePort=7946
level=info ts=2026-04-28T09:57:58.917859501Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-28T09:57:58.924796556Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T09:57:58.924824036Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T09:57:58.924844006Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.987062ms
level=error ts=2026-04-28T09:57:58.924851856Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-28T09:57:58.924864937Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-28T09:57:58.927004695Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:57:58.927026651Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T09:57:58.931374529Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-28T09:57:58.931387205Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-28T09:57:58.931387571Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-28T09:57:58.9314172Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-28T09:57:58.931428289Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-28T09:57:58.931456232Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-28T09:57:58.931527385Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-28T09:57:58.931534585Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-28T09:57:58.931725012Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-28T09:57:58.931733835Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-28T09:57:58.931793366Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-28T09:57:58.931806604Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-28T09:57:58.931835112Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-28T09:57:58.932012285Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-28T09:57:59.052208097Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=info ts=2026-04-28T09:57:59.052351272Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-28T09:57:59.176151489Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-28T09:57:59.176315327Z caller=loki.go:599 msg="Loki started" startup_time=324.283762ms
ts=2026-04-28T09:58:00.744131453Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:58:00.744172069Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T09:58:04.176321282Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-28T09:58:04.176401596Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-28T09:58:04.624087011Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:58:04.624119237Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:58:09.063965628Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T09:58:09.06399911Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T09:58:19.353837201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:58:22.470843043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:38974"
ts=2026-04-28T09:58:22.504730359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:58326"
ts=2026-04-28T09:58:24.21331984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.80:7946"
ts=2026-04-28T09:58:24.213424221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:37640"
ts=2026-04-28T09:58:24.215794053Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.73:7946: dial tcp 10.128.0.73:7946: connect: connection refused"
ts=2026-04-28T09:58:24.217210921Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
ts=2026-04-28T09:58:24.218225674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.68:7946"
ts=2026-04-28T09:58:24.220510714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.69:7946"
ts=2026-04-28T09:58:24.223264055Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.71:7946: dial tcp 10.128.0.71:7946: connect: connection refused"
level=info ts=2026-04-28T09:58:24.223277206Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=25.298407803s
ts=2026-04-28T09:58:26.006690574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:55948"
ts=2026-04-28T09:58:39.887760479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:53512"
level=debug ts=2026-04-28T09:58:40.317644818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:58:46.393727618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T09:58:57.622812165Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:59:03.48868268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49300"
ts=2026-04-28T09:59:16.396538711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T09:59:22.176273922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T09:59:27.256187921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:51844"
ts=2026-04-28T09:59:33.49152342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:48194"
ts=2026-04-28T09:59:39.89229559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:50358"
ts=2026-04-28T09:59:46.3995371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T09:59:53.142225627Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T09:59:53.144625494Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:00:09.895231695Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:46810"
ts=2026-04-28T10:00:16.402772153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:00:29.803611807Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T10:00:33.496155218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37290"
ts=2026-04-28T10:00:46.405644445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:01:03.499023747Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55076"
ts=2026-04-28T10:01:16.409167827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:01:38.920180549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T10:01:46.41213567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:01:57.265134739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:33028"
ts=2026-04-28T10:02:16.415374289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:02:27.268333459Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:44588"
ts=2026-04-28T10:02:33.504376092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45340"
level=debug ts=2026-04-28T10:02:44.182310725Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T10:02:46.418084758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:03:09.906235086Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39802"
ts=2026-04-28T10:03:16.421620827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:03:27.272384331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:47010"
ts=2026-04-28T10:03:33.509380393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50190"
level=debug ts=2026-04-28T10:03:39.158609307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T10:03:46.424436832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:04:09.911878787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:48380"
ts=2026-04-28T10:04:16.427120996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:04:41.761231097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T10:04:41.763046889Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:04:46.430094223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:05:16.432646792Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:05:38.696775297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:05:46.435416174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:06:03.519115542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59456"
ts=2026-04-28T10:06:16.438654852Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:06:28.628728219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:06:46.441276722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:06:57.282172369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:34588"
ts=2026-04-28T10:07:03.523086604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:32890"
level=debug ts=2026-04-28T10:07:13.239291602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:07:16.444401932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:07:27.28560849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:55468"
ts=2026-04-28T10:07:39.921656893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:36944"
ts=2026-04-28T10:07:46.447504056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=info ts=2026-04-28T10:08:04.176520162Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-28T10:08:04.180066568Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:08:04.182065037Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:08:04.18420535Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:08:04.184249563Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T10:08:04.184264918Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-28T10:08:05.112424902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:08:16.450707824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:08:46.453952035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:09:05.872712322Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:09:05.874618122Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:09:09.927432094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:57016"
ts=2026-04-28T10:09:16.457285593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:09:33.531585311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:54458"
ts=2026-04-28T10:09:39.930795706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39316"
ts=2026-04-28T10:09:46.460180446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:09:56.089284603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:10:03.534864094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47922"
ts=2026-04-28T10:10:16.463439698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:10:27.294514859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:60316"
ts=2026-04-28T10:10:33.537320549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56930"
ts=2026-04-28T10:10:39.936061275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:60238"
ts=2026-04-28T10:10:46.466695299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:10:56.666055656Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:11:09.938865876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:45270"
ts=2026-04-28T10:11:16.470091077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:11:33.541252711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55718"
level=debug ts=2026-04-28T10:11:39.636621964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:11:46.473029109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:12:03.544397708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37616"
ts=2026-04-28T10:12:09.943202976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:37666"
ts=2026-04-28T10:12:16.475980728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:12:18.665517107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:12:27.300626213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:53824"
ts=2026-04-28T10:12:33.54777562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49714"
ts=2026-04-28T10:12:46.479082796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:12:53.049439279Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:12:53.050986656Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:13:09.94810976Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:55016"
ts=2026-04-28T10:13:16.482453243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:13:33.552466534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55988"
ts=2026-04-28T10:13:46.485963934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:13:51.087599841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:13:57.305764083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42494"
ts=2026-04-28T10:14:16.488821762Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:14:27.308285386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:39272"
ts=2026-04-28T10:14:46.49157475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:14:52.896732777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:15:16.494247579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:15:27.312645277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:60264"
ts=2026-04-28T10:15:46.497475561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:15:47.926324799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:16:16.500912746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:16:27.31671949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:39224"
level=debug ts=2026-04-28T10:16:45.106435514Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:16:46.50450452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:17:09.961238767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:37706"
ts=2026-04-28T10:17:16.508000402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:17:27.320715678Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:36652"
level=debug ts=2026-04-28T10:17:45.047219699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:17:45.049306611Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:17:46.511035728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=error ts=2026-04-28T10:18:04.188568247Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:18:04.190754608Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:18:04.192744423Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:18:04.192776375Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:18:09.965326818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:53640"
ts=2026-04-28T10:18:16.514361552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:18:27.324963699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:50734"
level=debug ts=2026-04-28T10:18:43.312980083Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:18:46.517180303Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:19:09.970341934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:41352"
ts=2026-04-28T10:19:16.520405624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:19:20.033301751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:19:46.523611072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:20:00.569149452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:20:09.975591712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47738"
ts=2026-04-28T10:20:16.526568305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:20:33.575958321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:32908"
ts=2026-04-28T10:20:46.529345332Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:20:47.006389685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:20:57.333279571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:35982"
ts=2026-04-28T10:21:03.578546465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:35590"
ts=2026-04-28T10:21:16.532061064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:21:33.954361968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:21:33.957584017Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:21:39.981644391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40028"
ts=2026-04-28T10:21:46.535613307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:22:09.984460517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:45488"
ts=2026-04-28T10:22:16.538765858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:22:20.620348623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:22:27.338591944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:47238"
ts=2026-04-28T10:22:39.987193513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:49878"
ts=2026-04-28T10:22:46.541998151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:22:57.256090889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:23:16.544679659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:23:38.02496842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:23:46.547748168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:24:09.992593239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54412"
ts=2026-04-28T10:24:16.550371514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:24:27.345958753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:53516"
level=debug ts=2026-04-28T10:24:33.717261568Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:24:39.996215022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:51682"
ts=2026-04-28T10:24:46.553196358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:24:57.348658126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:48906"
ts=2026-04-28T10:25:09.9991451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:42574"
level=debug ts=2026-04-28T10:25:10.483443499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:25:10.485512865Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:25:16.556038174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:25:40.002041046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:51140"
ts=2026-04-28T10:25:46.558721628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:25:57.353769132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:58476"
level=debug ts=2026-04-28T10:25:59.6141326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:26:03.594543926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45150"
ts=2026-04-28T10:26:16.56135728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:26:27.356790547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:60834"
level=debug ts=2026-04-28T10:26:35.025572456Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:26:40.007382334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40892"
ts=2026-04-28T10:26:46.564562786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:26:57.360015004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:53412"
ts=2026-04-28T10:27:03.598637783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43416"
ts=2026-04-28T10:27:16.567278605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:27:21.276269277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:27:46.570609661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=error ts=2026-04-28T10:28:04.191087623Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:28:04.193616941Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:28:04.195959567Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:28:04.195985568Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:28:10.01852283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:49002"
level=debug ts=2026-04-28T10:28:13.282726368Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:28:16.580198279Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:28:27.365558991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:38566"
ts=2026-04-28T10:28:46.583679932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:28:55.128080908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:28:55.130852563Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:29:16.586541253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
ts=2026-04-28T10:29:27.371242097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:59724"
ts=2026-04-28T10:29:33.608013388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:51064"
ts=2026-04-28T10:29:40.032300472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:56152"
ts=2026-04-28T10:29:46.589254476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:29:51.221261168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:30:03.625787209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60886"
ts=2026-04-28T10:30:10.045934129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52420"
ts=2026-04-28T10:30:16.591895872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:30:27.37642298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:55536"
level=debug ts=2026-04-28T10:30:32.08011124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:30:46.594729951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:31:10.050735882Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:53718"
ts=2026-04-28T10:31:16.597354345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:31:34.577665315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:31:46.600193199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:32:03.633802931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:59180"
ts=2026-04-28T10:32:10.056138737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:59026"
ts=2026-04-28T10:32:16.603423076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6940eabf 10.128.0.72:7946"
level=debug ts=2026-04-28T10:32:31.363093818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:32:46.606977499Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:33:03.638247517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55016"
ts=2026-04-28T10:33:16.610528088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:33:27.313978918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:33:27.315822105Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:33:46.61393949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:34:03.643144582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:36232"
level=debug ts=2026-04-28T10:34:13.022645697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:34:14.186075473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:58634"
ts=2026-04-28T10:34:14.19415247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:58636"
ts=2026-04-28T10:34:16.617107342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:34:44.213327106Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-6940eabf as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T10:34:45.918065667Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6940eabf (timeout reached)"
ts=2026-04-28T10:34:46.620836683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:34:48.918866013Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6940eabf has failed, no acks received"
level=debug ts=2026-04-28T10:34:51.94744007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:35:16.624838825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:35:43.122660368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:58490"
ts=2026-04-28T10:35:46.62825543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e58b5fab 10.128.0.72:7946"
level=debug ts=2026-04-28T10:35:51.359178835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:35:57.394414557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:57978"
ts=2026-04-28T10:36:10.07206216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44486"
ts=2026-04-28T10:36:16.631148092Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:36:27.397532735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:57448"
level=debug ts=2026-04-28T10:36:29.55328824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:36:40.075288439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:51888"
ts=2026-04-28T10:36:43.126529762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47078"
ts=2026-04-28T10:36:46.633788859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:37:10.080477554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35038"
ts=2026-04-28T10:37:13.129851044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:44704"
ts=2026-04-28T10:37:16.636824462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e58b5fab 10.128.0.72:7946"
ts=2026-04-28T10:37:27.402340148Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:51838"
level=debug ts=2026-04-28T10:37:31.211179555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:37:31.213204474Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:37:40.083350469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47380"
ts=2026-04-28T10:37:46.640081407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=error ts=2026-04-28T10:38:04.190345377Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:38:04.192837575Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:38:04.194960302Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:38:04.19499046Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:38:16.643552477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e58b5fab 10.128.0.72:7946"
level=debug ts=2026-04-28T10:38:20.005698513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:38:40.087355732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54012"
ts=2026-04-28T10:38:46.646285866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:39:16.648835137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e58b5fab 10.128.0.72:7946"
level=debug ts=2026-04-28T10:39:18.767945268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:39:27.410175549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42354"
ts=2026-04-28T10:39:43.1389714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:36012"
ts=2026-04-28T10:39:46.651856661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:40:09.311965337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:40:10.092958404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:50760"
ts=2026-04-28T10:40:16.655280098Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:40:27.414839808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:57822"
ts=2026-04-28T10:40:46.658206561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:40:56.501266454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:40:57.418424628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:56452"
ts=2026-04-28T10:41:10.096710165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:50934"
ts=2026-04-28T10:41:10.918923875Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e58b5fab (timeout reached)"
ts=2026-04-28T10:41:13.918429937Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e58b5fab has failed, no acks received"
ts=2026-04-28T10:41:16.647330603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40554"
ts=2026-04-28T10:41:16.656560982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40564"
ts=2026-04-28T10:41:16.661036284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d8c132 10.128.0.72:7946"
ts=2026-04-28T10:41:20.918683723Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e58b5fab (timeout reached)"
ts=2026-04-28T10:41:23.918568361Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e58b5fab has failed, no acks received"
ts=2026-04-28T10:41:25.016301545Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-e58b5fab as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-28T10:41:42.594610537Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:41:42.596770538Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:41:46.66418959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d8c132 10.128.0.72:7946"
ts=2026-04-28T10:41:57.426851615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:43946"
ts=2026-04-28T10:42:16.667041211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:42:29.004530009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:42:40.101890376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:49630"
ts=2026-04-28T10:42:45.22904061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:51772"
ts=2026-04-28T10:42:46.66995161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d8c132 10.128.0.72:7946"
level=debug ts=2026-04-28T10:43:11.477058735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:43:16.67325138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:43:27.43120049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:50334"
ts=2026-04-28T10:43:45.233535389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:41852"
ts=2026-04-28T10:43:46.676705731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:43:59.604593733Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:44:16.688015173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d8c132 10.128.0.72:7946"
level=debug ts=2026-04-28T10:44:34.143434738Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:44:46.697113151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:45:10.115248133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43758"
ts=2026-04-28T10:45:15.239315893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60060"
ts=2026-04-28T10:45:16.700089807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:45:18.906363041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:45:18.908393843Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:45:27.438990041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:60902"
ts=2026-04-28T10:45:40.119574853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:56526"
ts=2026-04-28T10:45:46.703333584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:46:14.945328721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:46:16.70634032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:46:40.124033188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:59026"
ts=2026-04-28T10:46:46.709630537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d8c132 10.128.0.72:7946"
level=debug ts=2026-04-28T10:46:59.792651775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:47:10.126988483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:36226"
ts=2026-04-28T10:47:16.714623793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d8c132 10.128.0.72:7946"
level=debug ts=2026-04-28T10:47:38.471459764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:47:46.71743344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=error ts=2026-04-28T10:48:04.188936908Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:48:04.191012525Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:48:04.192948977Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:48:04.192972568Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:48:10.132237617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35782"
ts=2026-04-28T10:48:16.720718097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:48:20.432714925Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:48:45.251331054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:46430"
ts=2026-04-28T10:48:46.723977314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a0d8c132 10.128.0.72:7946"
ts=2026-04-28T10:49:16.727057668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T10:49:18.385896724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:49:18.387719038Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:49:25.918577039Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a0d8c132 (timeout reached)"
ts=2026-04-28T10:49:28.918548189Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a0d8c132 has failed, no acks received"
ts=2026-04-28T10:49:40.918954192Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-a0d8c132 (timeout reached)"
ts=2026-04-28T10:49:41.588332855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40904"
ts=2026-04-28T10:49:41.597100045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40916"
ts=2026-04-28T10:49:43.919276346Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-a0d8c132 has failed, no acks received"
ts=2026-04-28T10:49:46.729978384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:49:48.919051949Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-a0d8c132 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T10:50:10.138446544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:51852"
ts=2026-04-28T10:50:16.732953067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
level=debug ts=2026-04-28T10:50:18.849137099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:50:37.357066132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37412"
ts=2026-04-28T10:50:46.735675544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:51:06.350681254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:51:07.359648315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:33862"
ts=2026-04-28T10:51:16.738611255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
level=debug ts=2026-04-28T10:51:42.250654569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:51:46.741078801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
ts=2026-04-28T10:52:16.744339899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:52:40.147056433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:57890"
level=debug ts=2026-04-28T10:52:43.559686214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:52:46.747357742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
ts=2026-04-28T10:53:16.750196319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
level=debug ts=2026-04-28T10:53:23.5265803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:53:23.528152163Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:53:37.368564852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:41288"
ts=2026-04-28T10:53:46.753177997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:54:10.153286469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43810"
ts=2026-04-28T10:54:16.755859453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
level=debug ts=2026-04-28T10:54:22.014210319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:54:46.758419331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:55:00.648037699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:55:10.157305645Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43204"
ts=2026-04-28T10:55:16.760937375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:55:37.376068877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37416"
ts=2026-04-28T10:55:46.764159503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
level=debug ts=2026-04-28T10:55:49.871718929Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:56:07.379304643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:47920"
ts=2026-04-28T10:56:16.767079009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T10:56:46.769639339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:56:46.983227107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:57:16.772995497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:57:39.065168748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T10:57:39.066702229Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:57:40.165104249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:33630"
ts=2026-04-28T10:57:46.776280049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=error ts=2026-04-28T10:58:04.188649602Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:58:04.190696084Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:58:04.19249038Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T10:58:04.192520276Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T10:58:10.168203469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:42708"
ts=2026-04-28T10:58:16.779230406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T10:58:31.211695355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:58:40.17113456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:42996"
ts=2026-04-28T10:58:46.782026326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
ts=2026-04-28T10:59:10.173702964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:55742"
ts=2026-04-28T10:59:16.785137636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
level=debug ts=2026-04-28T10:59:16.848084025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T10:59:27.481949526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:54638"
ts=2026-04-28T10:59:46.788312893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T10:59:57.48520345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:37182"
ts=2026-04-28T11:00:10.177979766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:60414"
ts=2026-04-28T11:00:16.791391121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-23afbed5 10.128.0.72:7946"
level=debug ts=2026-04-28T11:00:17.559618203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:00:21.790033804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55776"
ts=2026-04-28T11:00:21.797555462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55782"
ts=2026-04-28T11:00:35.919452094Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-23afbed5 (timeout reached)"
ts=2026-04-28T11:00:38.919671639Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-23afbed5 has failed, no acks received"
ts=2026-04-28T11:00:40.18124871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:57984"
ts=2026-04-28T11:00:46.794465983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:00:49.811562101Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-23afbed5 as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-28T11:00:53.841265551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:00:57.489690973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:59084"
ts=2026-04-28T11:01:16.797409294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:01:40.184796021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:45750"
ts=2026-04-28T11:01:46.800263758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T11:01:54.625784609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:01:54.627999027Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:01:57.493520219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:45648"
ts=2026-04-28T11:02:16.803218026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:02:27.496712942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:52600"
level=debug ts=2026-04-28T11:02:29.276313762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:02:40.188147216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34900"
ts=2026-04-28T11:02:46.80616959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:02:57.499364515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:55876"
ts=2026-04-28T11:03:16.809358362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T11:03:31.607814993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:03:46.811747591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:03:57.503643384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:41768"
ts=2026-04-28T11:03:59.404886895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:35706"
ts=2026-04-28T11:04:16.815096567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T11:04:27.506986853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:58784"
level=debug ts=2026-04-28T11:04:32.467003924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:04:46.818101182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T11:05:16.820870625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
level=debug ts=2026-04-28T11:05:21.876617886Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:05:46.823782413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
ts=2026-04-28T11:05:59.411439564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55646"
ts=2026-04-28T11:06:16.826625447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T11:06:19.159636104Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:06:19.161349348Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:06:46.829411537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
ts=2026-04-28T11:07:10.201246623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44952"
level=debug ts=2026-04-28T11:07:12.778035144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:07:16.831742593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
ts=2026-04-28T11:07:27.517957949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:53736"
ts=2026-04-28T11:07:29.416007603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:36024"
ts=2026-04-28T11:07:46.834519657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:07:59.418430181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43150"
level=error ts=2026-04-28T11:08:04.187482902Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:08:04.189228684Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:08:04.190884869Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:08:04.190919474Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T11:08:05.555637435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:08:16.837166781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
ts=2026-04-28T11:08:46.839955736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
level=debug ts=2026-04-28T11:08:49.755294293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:08:59.423235429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43978"
ts=2026-04-28T11:09:16.843193244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
ts=2026-04-28T11:09:27.525262411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42612"
ts=2026-04-28T11:09:29.426498859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:37856"
level=debug ts=2026-04-28T11:09:31.202864799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:09:46.846089702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
ts=2026-04-28T11:09:57.528037385Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:56904"
ts=2026-04-28T11:10:16.849052232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T11:10:28.984588502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:10:28.98624148Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:10:40.210998577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40524"
ts=2026-04-28T11:10:46.852480412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T11:10:57.532084409Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:54442"
ts=2026-04-28T11:10:59.431559679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55018"
ts=2026-04-28T11:11:16.855838187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T11:11:20.485004105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:11:40.21518631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47288"
ts=2026-04-28T11:11:46.85899676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
ts=2026-04-28T11:11:57.536169911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:34280"
level=debug ts=2026-04-28T11:12:16.188163895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:12:16.862111446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T11:12:29.436329805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:42604"
ts=2026-04-28T11:12:40.219301152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:37140"
ts=2026-04-28T11:12:46.864749767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:12:59.439164043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:57294"
ts=2026-04-28T11:13:10.222389366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:36982"
level=debug ts=2026-04-28T11:13:11.284204648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:13:16.868041624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-1996bf43 10.128.0.72:7946"
ts=2026-04-28T11:13:29.441947393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:44604"
ts=2026-04-28T11:13:46.871509918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:13:52.872032838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:38760"
ts=2026-04-28T11:13:52.879573023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:38766"
ts=2026-04-28T11:13:57.543536308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:58534"
level=debug ts=2026-04-28T11:13:59.56270326Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:14:00.919564238Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7520cbf9 (timeout reached)"
ts=2026-04-28T11:14:03.919231955Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7520cbf9 has failed, no acks received"
ts=2026-04-28T11:14:06.659565104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56302"
ts=2026-04-28T11:14:06.669140622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:56312"
ts=2026-04-28T11:14:16.874495919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:14:20.918938886Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-1996bf43 (timeout reached)"
ts=2026-04-28T11:14:23.918937059Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-1996bf43 has failed, no acks received"
ts=2026-04-28T11:14:23.92001452Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-7520cbf9 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T11:14:25.028926004Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-1996bf43 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T11:14:25.919782747Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7520cbf9 (timeout reached)"
ts=2026-04-28T11:14:27.546986746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:42364"
ts=2026-04-28T11:14:28.920004929Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7520cbf9 has failed, no acks received"
ts=2026-04-28T11:14:40.227310785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34266"
ts=2026-04-28T11:14:46.876995923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T11:14:49.264457622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:14:49.266194362Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:15:16.880094011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:15:46.882722007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T11:15:47.951312471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:15:57.553253894Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:35506"
ts=2026-04-28T11:16:10.232965553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43906"
ts=2026-04-28T11:16:16.886285069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-942bd350 10.128.0.72:7946"
ts=2026-04-28T11:16:27.556386219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:40510"
level=debug ts=2026-04-28T11:16:35.472607182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:16:46.889176431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:17:16.891625629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T11:17:26.510941431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:17:27.561335927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:57336"
ts=2026-04-28T11:17:40.238992011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:36826"
ts=2026-04-28T11:17:46.894986481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-942bd350 10.128.0.72:7946"
ts=2026-04-28T11:17:57.564305497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:57244"
level=debug ts=2026-04-28T11:18:03.413861635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:18:04.186558123Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:18:04.188325632Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:18:04.189984549Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-28T11:18:04.19001062Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:18:16.89819277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-942bd350 10.128.0.72:7946"
ts=2026-04-28T11:18:27.567641792Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:46792"
level=debug ts=2026-04-28T11:18:38.490688455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:18:38.492502416Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:18:46.901305929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:18:57.571003634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:54868"
ts=2026-04-28T11:19:16.903875257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:19:27.573839588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:40434"
level=debug ts=2026-04-28T11:19:34.985046618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:19:40.246001546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54248"
ts=2026-04-28T11:19:45.920029529Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-942bd350 (timeout reached)"
ts=2026-04-28T11:19:46.907018373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:19:48.919264092Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-942bd350 has failed, no acks received"
ts=2026-04-28T11:19:57.576940054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:59582"
ts=2026-04-28T11:20:00.918953495Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-942bd350 (timeout reached)"
ts=2026-04-28T11:20:03.91889072Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-942bd350 has failed, no acks received"
ts=2026-04-28T11:20:08.581544538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34848"
ts=2026-04-28T11:20:08.590034249Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:34856"
ts=2026-04-28T11:20:08.919544799Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-942bd350 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-28T11:20:10.248849557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47232"
level=debug ts=2026-04-28T11:20:14.307051099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:20:16.910377784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54a124b7 10.128.0.72:7946"
ts=2026-04-28T11:20:40.252082043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52502"
ts=2026-04-28T11:20:46.913104149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
level=debug ts=2026-04-28T11:20:58.250484265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:21:16.916449288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54a124b7 10.128.0.72:7946"
ts=2026-04-28T11:21:36.313505282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:39002"
level=debug ts=2026-04-28T11:21:42.31608691Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:21:46.919958075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T11:22:06.316390028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:46048"
ts=2026-04-28T11:22:16.923642434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T11:22:24.348493679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T11:22:24.350487345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T11:22:46.927005666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
ts=2026-04-28T11:22:57.585234829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:58798"
ts=2026-04-28T11:23:06.320201714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60478"
ts=2026-04-28T11:23:16.929498132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-gs8xg-f72685a7 10.128.0.69:7946"
level=debug ts=2026-04-28T11:23:24.461454224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:23:27.587772851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:43236"
ts=2026-04-28T11:23:46.93223862Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-54a124b7 10.128.0.72:7946"
level=debug ts=2026-04-28T11:24:11.880928498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T11:24:16.934798477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-zncdj-affe5d0c 10.128.0.68:7946"
ts=2026-04-28T11:24:27.59160846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.68:43756"