level=info ts=2026-04-20T09:00:23.210686309Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T09:00:23.210721965Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-20T09:00:23.212643021Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T09:00:23.213006634Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T09:00:23.213083737Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.74 inf=eth0
level=debug ts=2026-04-20T09:00:23.214180442Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T09:00:23.214255694Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.74 inf=eth0
level=info ts=2026-04-20T09:00:23.215000647Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-58b0533d
ts=2026-04-20T09:00:23.215078849Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T09:00:23.215094864Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.74 advertisePort=7946
level=debug ts=2026-04-20T09:00:23.215118625Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.74 advertisePort=7946
level=info ts=2026-04-20T09:00:23.215275269Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-20T09:00:23.221223232Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T09:00:23.22124232Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T09:00:23.221255016Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.982553ms
level=error ts=2026-04-20T09:00:23.221265022Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T09:00:23.221279622Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-20T09:00:23.224133423Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T09:00:23.224151931Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T09:00:23.228477935Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-20T09:00:23.2284977Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-20T09:00:23.228493365Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-20T09:00:23.228502716Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T09:00:23.228584665Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-20T09:00:23.228586898Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=info ts=2026-04-20T09:00:23.22859558Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-20T09:00:23.228668102Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-20T09:00:23.22868394Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=info ts=2026-04-20T09:00:23.228714409Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-20T09:00:23.228764949Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-20T09:00:23.228766742Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-20T09:00:23.228784498Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T09:00:23.228942633Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-20T09:00:23.228950999Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-20T09:00:23.228962267Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T09:00:24.229941787Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-20T09:00:24.366200604Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-20T09:00:24.366248213Z caller=loki.go:599 msg="Loki started" startup_time=1.221934451s
ts=2026-04-20T09:00:24.468557103Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T09:00:24.46858186Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:00:26.980195281Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T09:00:26.980222016Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T09:00:29.366854859Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-20T09:00:29.366909831Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-20T09:00:33.503029511Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T09:00:33.503051694Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T09:00:38.206636329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:00:47.071923567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.70:7946"
ts=2026-04-20T09:00:47.075410245Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.72:7946: dial tcp 10.128.0.72:7946: connect: connection refused"
level=info ts=2026-04-20T09:00:47.075426589Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=23.854140628s
ts=2026-04-20T09:00:49.642797208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51050"
level=debug ts=2026-04-20T09:00:57.975366614Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:01:19.233304259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:01:24.332151296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:01:40.098485174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:33796"
level=debug ts=2026-04-20T09:01:41.033493142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:01:49.234694275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:02:08.317399575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T09:02:08.319126991Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:02:19.237149826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:02:49.240359264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:02:50.624310379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50368"
level=debug ts=2026-04-20T09:02:54.211651419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:03:01.807417332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:51548"
ts=2026-04-20T09:03:19.242938169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:03:20.626088991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:41848"
ts=2026-04-20T09:03:40.112515219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56836"
ts=2026-04-20T09:03:49.244766286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:04:01.812521277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:33112"
level=debug ts=2026-04-20T09:04:05.242641342Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:04:19.247173496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:04:49.249051043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:05:01.814865791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42070"
level=debug ts=2026-04-20T09:05:05.494627979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:05:19.250579603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:05:40.126496511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56406"
ts=2026-04-20T09:05:49.252058331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:06:12.074289818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-20T09:06:19.254780895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:06:49.257241402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:07:19.258175484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:07:29.219507315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T09:07:29.221281146Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:07:49.259848311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:08:01.826341945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49850"
level=debug ts=2026-04-20T09:08:17.657499437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:08:19.262062691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:08:20.648413103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:55410"
ts=2026-04-20T09:08:49.264666221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
level=debug ts=2026-04-20T09:09:01.72364098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:09:19.266239898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:09:20.653120537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34788"
level=debug ts=2026-04-20T09:09:47.068647078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:09:49.269063435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:10:19.271383536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:10:20.655362335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:49888"
level=info ts=2026-04-20T09:10:29.366925245Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-20T09:10:29.37078567Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:10:29.372860669Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:10:29.374939086Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:10:29.37496628Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T09:10:29.374982312Z caller=tables_manager.go:145 msg="compactor started"
level=debug ts=2026-04-20T09:10:34.507211356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:10:49.272787333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:11:19.275575626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:11:20.657505908Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:33788"
level=debug ts=2026-04-20T09:11:26.217234583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:11:26.218956763Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:11:31.838959643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44038"
ts=2026-04-20T09:11:49.279876953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:12:10.169843617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46566"
level=debug ts=2026-04-20T09:12:11.366124951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:12:19.282397417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:12:20.660165607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53200"
level=debug ts=2026-04-20T09:12:48.275328491Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:12:49.285297244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:13:19.287227293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
level=debug ts=2026-04-20T09:13:39.282758659Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:13:40.179366545Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:39230"
ts=2026-04-20T09:13:49.288754967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:14:10.182974388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45932"
ts=2026-04-20T09:14:19.290232384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
level=debug ts=2026-04-20T09:14:40.941793487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:14:49.292663101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:15:10.189460205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:41390"
ts=2026-04-20T09:15:19.295878544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:15:31.860432512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:42982"
level=debug ts=2026-04-20T09:15:32.516583044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:15:32.518205476Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:15:49.298368014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:15:50.676522731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35138"
ts=2026-04-20T09:16:01.862194589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:45026"
ts=2026-04-20T09:16:10.195170603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:49670"
level=debug ts=2026-04-20T09:16:15.607175523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:16:19.299587252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:16:49.300838867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:16:50.679566421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:34532"
level=debug ts=2026-04-20T09:17:14.852529629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:17:19.303392475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:17:40.204784552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45134"
ts=2026-04-20T09:17:49.306146816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:18:01.869194228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48028"
level=debug ts=2026-04-20T09:18:05.648564026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:18:19.307124023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:18:20.687541216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:46524"
ts=2026-04-20T09:18:49.308497706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:18:50.689350505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36898"
level=debug ts=2026-04-20T09:19:04.411571583Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:19:10.215615543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51110"
ts=2026-04-20T09:19:19.309370641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:19:20.691101417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58876"
ts=2026-04-20T09:19:31.874675756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46780"
level=debug ts=2026-04-20T09:19:39.028592846Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:19:39.030217037Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:19:49.310519594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:19:50.692255072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:47490"
level=debug ts=2026-04-20T09:20:14.632081552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:20:19.311750581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=error ts=2026-04-20T09:20:29.379178796Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:20:29.380969418Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:20:29.382622733Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:20:29.382668815Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:20:31.879142964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:52630"
ts=2026-04-20T09:20:49.313305876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:20:50.694687658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51276"
level=debug ts=2026-04-20T09:20:58.611045911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:21:19.314425949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:21:20.696251464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39856"
level=debug ts=2026-04-20T09:21:45.87037671Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:21:49.315803887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:22:19.318378083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:22:20.701107189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57418"
level=debug ts=2026-04-20T09:22:33.757590867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:22:49.321113876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:23:01.889164178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58144"
ts=2026-04-20T09:23:10.242415738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38624"
ts=2026-04-20T09:23:19.32274693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:23:20.706456865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:60932"
level=debug ts=2026-04-20T09:23:25.079162513Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:23:25.080745115Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:23:40.246089693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:58556"
ts=2026-04-20T09:23:49.32420955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
level=debug ts=2026-04-20T09:24:18.15130703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:24:19.32698415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:24:49.3291518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:25:00.95785251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:25:19.331375866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:25:40.260902161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51994"
level=debug ts=2026-04-20T09:25:46.74703739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:25:49.333332155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:26:19.334291123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:26:49.335244408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:26:49.441105093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:26:50.726018692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58446"
ts=2026-04-20T09:27:19.336488285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
level=debug ts=2026-04-20T09:27:45.187632295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:27:45.189299102Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:27:49.339130183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:28:01.908081754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54270"
ts=2026-04-20T09:28:19.341441363Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
level=debug ts=2026-04-20T09:28:24.53513578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:28:49.343180977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:29:01.91294441Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53236"
level=debug ts=2026-04-20T09:29:04.011378595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:29:19.345676596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:29:20.735413613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53968"
ts=2026-04-20T09:29:31.914295631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:36176"
ts=2026-04-20T09:29:40.294182492Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:53670"
ts=2026-04-20T09:29:49.348344833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:29:50.736603078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57946"
level=debug ts=2026-04-20T09:29:51.941107207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:30:10.298489733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:52838"
ts=2026-04-20T09:30:19.350110636Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=error ts=2026-04-20T09:30:29.391283141Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:30:29.394930953Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:30:29.401449382Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:30:29.401498488Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T09:30:35.953266201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:30:40.301750719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:58116"
ts=2026-04-20T09:30:49.351904269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:30:50.739740965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39694"
ts=2026-04-20T09:31:10.305163334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:44816"
ts=2026-04-20T09:31:19.353254353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
ts=2026-04-20T09:31:20.741396573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50706"
level=debug ts=2026-04-20T09:31:27.707927031Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:31:27.710131006Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:31:31.926218284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53234"
ts=2026-04-20T09:31:49.355476236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:31:50.742523045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:46474"
level=debug ts=2026-04-20T09:32:04.817171059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:32:10.311714117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:34582"
ts=2026-04-20T09:32:19.359333644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:32:20.743496366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:58488"
ts=2026-04-20T09:32:31.929367277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:40046"
ts=2026-04-20T09:32:40.314525096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:40278"
ts=2026-04-20T09:32:49.361980059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:32:53.053222845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:33:19.363339194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:33:20.748404162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:32874"
ts=2026-04-20T09:33:31.932074575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:58644"
level=debug ts=2026-04-20T09:33:33.728214574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:33:49.364128193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:34:01.933120647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56332"
ts=2026-04-20T09:34:19.36560323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:34:23.818114805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:34:49.366685774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:35:00.155852916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:35:00.157811024Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:35:01.936055014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:41582"
ts=2026-04-20T09:35:19.369196874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:35:31.937990784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46222"
ts=2026-04-20T09:35:49.371737431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:35:50.760498144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:40914"
level=debug ts=2026-04-20T09:35:56.147720319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:36:01.938915691Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:41240"
ts=2026-04-20T09:36:10.33795607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:59700"
ts=2026-04-20T09:36:19.377142495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:36:40.341366391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:52896"
ts=2026-04-20T09:36:49.380109418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-e10736e3 10.129.0.73:7946"
level=debug ts=2026-04-20T09:36:55.504467745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:37:10.345193162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:38284"
ts=2026-04-20T09:37:19.382878467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:37:28.752377815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43554"
ts=2026-04-20T09:37:28.763168499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43570"
ts=2026-04-20T09:37:45.215721884Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e10736e3 (timeout reached)"
ts=2026-04-20T09:37:48.216410268Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e10736e3 has failed, no acks received"
ts=2026-04-20T09:37:49.384893739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:37:50.01419381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:37:50.773081055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:42466"
ts=2026-04-20T09:38:00.216433444Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-e10736e3 (timeout reached)"
ts=2026-04-20T09:38:03.216105657Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-e10736e3 has failed, no acks received"
ts=2026-04-20T09:38:10.352463021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56334"
ts=2026-04-20T09:38:19.387550838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T09:38:31.272249425Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:38:49.390773977Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:39:10.359248278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55216"
ts=2026-04-20T09:39:19.392878082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:39:23.361058811Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:39:23.362921996Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:39:28.038832395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:50378"
ts=2026-04-20T09:39:40.362187169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46514"
ts=2026-04-20T09:39:49.393813405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:40:10.365867625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47504"
ts=2026-04-20T09:40:19.395025651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b813a79 10.129.0.73:7946"
level=debug ts=2026-04-20T09:40:23.706031825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:40:29.38209246Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:40:29.384267035Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:40:29.386110565Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:40:29.386138815Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:40:49.396268006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b813a79 10.129.0.73:7946"
ts=2026-04-20T09:40:58.042552077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43852"
level=debug ts=2026-04-20T09:41:11.611161249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:41:19.399130825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:41:49.402359168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T09:41:56.664945292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:42:19.405145162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b813a79 10.129.0.73:7946"
ts=2026-04-20T09:42:20.793579625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50088"
ts=2026-04-20T09:42:49.406942325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T09:42:52.800551665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:42:58.053686156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60932"
ts=2026-04-20T09:43:19.409150005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:43:36.768800557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:43:36.770870008Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:43:49.410779024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:44:10.402981477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:39748"
ts=2026-04-20T09:44:19.412118911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b813a79 10.129.0.73:7946"
level=debug ts=2026-04-20T09:44:20.306331601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:44:28.059881074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46592"
ts=2026-04-20T09:44:49.413628673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b813a79 10.129.0.73:7946"
level=debug ts=2026-04-20T09:45:00.728724972Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:45:10.410200018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:43888"
ts=2026-04-20T09:45:19.414655582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b813a79 10.129.0.73:7946"
level=debug ts=2026-04-20T09:45:38.861695324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:45:49.416447711Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-7b813a79 10.129.0.73:7946"
ts=2026-04-20T09:46:19.41910443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T09:46:37.794135015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:46:49.421322972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:47:00.215765192Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7b813a79 (timeout reached)"
ts=2026-04-20T09:47:03.216332997Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7b813a79 has failed, no acks received"
ts=2026-04-20T09:47:05.216789665Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7b813a79 (timeout reached)"
ts=2026-04-20T09:47:08.217331341Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7b813a79 has failed, no acks received"
ts=2026-04-20T09:47:10.424343386Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55240"
ts=2026-04-20T09:47:13.868676031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37398"
ts=2026-04-20T09:47:13.884367793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:37406"
level=debug ts=2026-04-20T09:47:18.357744308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:47:18.359640281Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:47:19.422471627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:47:20.216331772Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7b813a79 (timeout reached)"
ts=2026-04-20T09:47:23.215954244Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7b813a79 has failed, no acks received"
ts=2026-04-20T09:47:23.21729909Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-7b813a79 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-20T09:47:25.216867911Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-7b813a79 (timeout reached)"
ts=2026-04-20T09:47:28.216957955Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-7b813a79 has failed, no acks received"
ts=2026-04-20T09:47:49.424481451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T09:47:58.423360508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:48:10.430507832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:42500"
ts=2026-04-20T09:48:19.427138879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:48:40.433960187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46308"
ts=2026-04-20T09:48:49.428512886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:48:50.828239491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:38396"
level=debug ts=2026-04-20T09:48:59.976842483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:49:19.43118509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:49:40.440197974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47336"
level=debug ts=2026-04-20T09:49:44.38514792Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:49:49.433674567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b071e36d 10.129.0.73:7946"
ts=2026-04-20T09:50:19.435397427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
level=debug ts=2026-04-20T09:50:27.977137505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:50:29.378146837Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:50:29.380510943Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:50:29.382410949Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T09:50:29.382437287Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:50:41.545742052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47232"
ts=2026-04-20T09:50:49.436709426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b071e36d 10.129.0.73:7946"
level=debug ts=2026-04-20T09:51:04.58565494Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:51:04.587749489Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:51:10.451142412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:36328"
ts=2026-04-20T09:51:11.547503593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34770"
ts=2026-04-20T09:51:19.438241694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b071e36d 10.129.0.73:7946"
ts=2026-04-20T09:51:40.4547479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:35064"
ts=2026-04-20T09:51:41.548929033Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:56824"
ts=2026-04-20T09:51:49.439731571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b071e36d 10.129.0.73:7946"
ts=2026-04-20T09:51:50.840067002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35388"
level=debug ts=2026-04-20T09:52:00.225882477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:52:11.549792269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:46832"
ts=2026-04-20T09:52:19.441551476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b071e36d 10.129.0.73:7946"
ts=2026-04-20T09:52:40.46153674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:60520"
ts=2026-04-20T09:52:41.551009881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:40464"
ts=2026-04-20T09:52:49.443874874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:52:50.844142773Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:43796"
level=debug ts=2026-04-20T09:52:58.892640205Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:53:10.465246825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:46956"
ts=2026-04-20T09:53:11.55240091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43430"
ts=2026-04-20T09:53:19.447510378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T09:53:46.336720939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:53:49.449261635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b071e36d 10.129.0.73:7946"
ts=2026-04-20T09:54:10.471318886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55562"
ts=2026-04-20T09:54:19.450844016Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b071e36d 10.129.0.73:7946"
level=debug ts=2026-04-20T09:54:30.703552742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:54:49.45235233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:55:19.453303219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-b071e36d 10.129.0.73:7946"
ts=2026-04-20T09:55:20.853117936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:37590"
level=debug ts=2026-04-20T09:55:32.630862574Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T09:55:32.632362Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T09:55:41.5622406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53132"
ts=2026-04-20T09:55:49.455331635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:56:11.56378946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48830"
ts=2026-04-20T09:56:19.458192502Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:56:20.85570669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:57304"
ts=2026-04-20T09:56:30.216586386Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b071e36d (timeout reached)"
level=debug ts=2026-04-20T09:56:31.082138584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:56:33.216734518Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b071e36d has failed, no acks received"
ts=2026-04-20T09:56:45.217078783Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-b071e36d (timeout reached)"
ts=2026-04-20T09:56:48.217116439Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-b071e36d has failed, no acks received"
ts=2026-04-20T09:56:48.711099537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54456"
ts=2026-04-20T09:56:48.722356108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54466"
ts=2026-04-20T09:56:49.460965138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68636a1d 10.129.0.73:7946"
ts=2026-04-20T09:56:53.217548352Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-b071e36d as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-20T09:57:10.492060776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:45704"
ts=2026-04-20T09:57:19.463324331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T09:57:20.945382496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:57:40.49613222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:41032"
ts=2026-04-20T09:57:49.465876863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68636a1d 10.129.0.73:7946"
ts=2026-04-20T09:57:50.86284352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:39916"
level=debug ts=2026-04-20T09:58:14.702664682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:58:19.4667089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68636a1d 10.129.0.73:7946"
ts=2026-04-20T09:58:49.468813633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T09:59:01.262472515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:49888"
level=debug ts=2026-04-20T09:59:08.115638358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T09:59:10.506632664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:55784"
ts=2026-04-20T09:59:19.471229257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T09:59:31.26358719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:41268"
ts=2026-04-20T09:59:49.47225652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T10:00:01.265042257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:57432"
level=debug ts=2026-04-20T10:00:09.164708993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T10:00:09.167416062Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T10:00:19.473693758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68636a1d 10.129.0.73:7946"
ts=2026-04-20T10:00:20.873042961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:35984"
level=error ts=2026-04-20T10:00:29.37912679Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T10:00:29.380979503Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T10:00:29.382838025Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T10:00:29.382864583Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T10:00:31.266641952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:60766"
ts=2026-04-20T10:00:40.516509234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:37952"
ts=2026-04-20T10:00:49.475496109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68636a1d 10.129.0.73:7946"
level=debug ts=2026-04-20T10:00:56.213167547Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:01:01.267482901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:43616"
ts=2026-04-20T10:01:19.476948764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T10:01:31.269116887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:44286"
level=debug ts=2026-04-20T10:01:47.38301545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:01:49.478314766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T10:02:19.480019164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68636a1d 10.129.0.73:7946"
ts=2026-04-20T10:02:20.882689927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:50158"
level=debug ts=2026-04-20T10:02:23.216426682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:02:49.48158549Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-68636a1d 10.129.0.73:7946"
ts=2026-04-20T10:02:50.884512842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:38116"
ts=2026-04-20T10:03:01.275524398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:47830"
ts=2026-04-20T10:03:19.483092489Z caller=memberlist_logger.go:74 level=error msg="Push/Pull with loki-ingester-0-68636a1d failed: dial tcp 10.129.0.73:7946: connect: connection refused"
ts=2026-04-20T10:03:20.216855889Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-68636a1d (timeout reached)"
ts=2026-04-20T10:03:20.886015985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:53770"
level=debug ts=2026-04-20T10:03:22.017658593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:03:23.216640478Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-68636a1d has failed, no acks received"
ts=2026-04-20T10:03:35.216929485Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-68636a1d (timeout reached)"
ts=2026-04-20T10:03:38.216285135Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-68636a1d has failed, no acks received"
ts=2026-04-20T10:03:40.537338679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:51530"
ts=2026-04-20T10:03:42.368646809Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-68636a1d as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-20T10:03:49.4843702Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T10:04:07.589421764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48608"
ts=2026-04-20T10:04:07.600768555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:48622"
level=debug ts=2026-04-20T10:04:07.671279503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T10:04:07.673600931Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T10:04:10.541080713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:42014"
ts=2026-04-20T10:04:19.485288264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T10:04:49.487360138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T10:05:09.193209333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:05:19.48957678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T10:05:20.893539453Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59810"
ts=2026-04-20T10:05:44.3883218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:38976"
ts=2026-04-20T10:05:49.491429966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ffa049ee 10.129.0.73:7946"
ts=2026-04-20T10:05:50.89529987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:36056"
level=debug ts=2026-04-20T10:06:11.1665281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:06:19.492992458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nnn7p-24e8534a 10.129.0.70:7946"
ts=2026-04-20T10:06:44.392541457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:34894"
ts=2026-04-20T10:06:49.495645469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T10:07:09.768151599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:07:10.558614586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:36106"
ts=2026-04-20T10:07:19.499132484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T10:07:44.395256425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:54724"
ts=2026-04-20T10:07:49.501542783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ffa049ee 10.129.0.73:7946"
level=debug ts=2026-04-20T10:07:58.214099149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:08:19.503719837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T10:08:44.400364643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.73:53458"
level=debug ts=2026-04-20T10:08:46.054771264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T10:08:46.056343178Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T10:08:49.507282244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T10:08:50.910218746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:59218"
ts=2026-04-20T10:09:19.509814774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ffa049ee 10.129.0.73:7946"
ts=2026-04-20T10:09:40.575931729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:56280"
level=debug ts=2026-04-20T10:09:46.110633863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:09:49.511982186Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T10:10:10.579385547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:37476"
ts=2026-04-20T10:10:19.515852314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T10:10:21.557658648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T10:10:29.380494328Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T10:10:29.382211768Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T10:10:29.383883243Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-20T10:10:29.383911043Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-20T10:10:40.583040764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:35518"
ts=2026-04-20T10:10:49.519588117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T10:10:56.154498761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:11:19.522370568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
level=debug ts=2026-04-20T10:11:37.551583623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T10:11:40.589953528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.70:47156"
ts=2026-04-20T10:11:49.525607351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-djq7f-bfb230fc 10.128.0.70:7946"
ts=2026-04-20T10:11:50.925506676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.70:51390"
ts=2026-04-20T10:12:19.527561454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-ffa049ee 10.129.0.73:7946"
level=debug ts=2026-04-20T10:12:30.429450001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T10:12:30.431045714Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"