level=info ts=2026-04-19T14:32:01.903201094Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-19T14:32:01.903254112Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-19T14:32:01.904754131Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-19T14:32:01.905187807Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T14:32:01.905294772Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.58 inf=eth0
level=debug ts=2026-04-19T14:32:01.906199001Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-19T14:32:01.906276225Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.58 inf=eth0
level=info ts=2026-04-19T14:32:01.906412494Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-b0837a67
ts=2026-04-19T14:32:01.906489645Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-19T14:32:01.90650579Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.58 advertisePort=7946
level=debug ts=2026-04-19T14:32:01.906528618Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.58 advertisePort=7946
level=info ts=2026-04-19T14:32:01.90668492Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-19T14:32:01.913611281Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T14:32:01.913630715Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-19T14:32:01.913642912Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=6.960977ms
level=error ts=2026-04-19T14:32:01.913652709Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-19T14:32:01.913667873Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-19T14:32:01.917070367Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T14:32:01.917094546Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T14:32:01.922385273Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-19T14:32:01.922489098Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-19T14:32:01.922493421Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-19T14:32:01.922542784Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-19T14:32:01.922629042Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-19T14:32:01.922874116Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-19T14:32:01.92295782Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-04-19T14:32:01.922964537Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-19T14:32:01.922970254Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-19T14:32:01.922975443Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-19T14:32:01.92304383Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-19T14:32:01.92309727Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-19T14:32:01.923116445Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-19T14:32:01.923266918Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-19T14:32:01.92327482Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-19T14:32:01.923289582Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-19T14:32:02.923553156Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-19T14:32:02.923579134Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-19T14:32:02.923599344Z caller=loki.go:599 msg="Loki started" startup_time=1.088051993s
ts=2026-04-19T14:32:03.357778814Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T14:32:03.357811117Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:32:06.162177983Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T14:32:06.162213779Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-19T14:32:07.924413008Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-19T14:32:07.924471071Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-19T14:32:11.842910703Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-19T14:32:11.842939533Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-19T14:32:15.912815289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:32:21.449284068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.66:7946"
ts=2026-04-19T14:32:21.452399547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.67:7946"
level=info ts=2026-04-19T14:32:21.453800578Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=19.540127118s
ts=2026-04-19T14:32:26.603169521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:60026"
level=debug ts=2026-04-19T14:32:32.959566333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:32:34.969122886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:32:55.062815162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:33:04.970797535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:33:14.856273186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:43134"
level=debug ts=2026-04-19T14:33:20.107171714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:33:34.971778593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:33:47.058032649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T14:33:47.059642345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:34:04.9738831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T14:34:18.102800301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:34:34.977152399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:34:44.861103493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41988"
ts=2026-04-19T14:35:04.98045298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:35:08.743678843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:58816"
level=debug ts=2026-04-19T14:35:20.771962376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:35:31.269687058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:49006"
ts=2026-04-19T14:35:34.98277114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:36:01.270880202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:56442"
ts=2026-04-19T14:36:04.985080848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:36:08.748603553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51082"
level=debug ts=2026-04-19T14:36:17.745733058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:36:31.272380353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:51776"
ts=2026-04-19T14:36:34.988164838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:36:44.871157111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52656"
ts=2026-04-19T14:37:04.992100308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:37:14.873801836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:37564"
level=debug ts=2026-04-19T14:37:24.220814759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-19T14:37:31.276589135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:51684"
ts=2026-04-19T14:37:34.995076114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:37:44.876360872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:34544"
ts=2026-04-19T14:38:04.998177244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T14:38:15.708957168Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-19T14:38:15.710487145Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:38:35.001778656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:39:01.284959054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44436"
ts=2026-04-19T14:39:05.004045503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:39:08.762480579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60972"
level=debug ts=2026-04-19T14:39:11.726199685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:39:31.286246447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:54422"
ts=2026-04-19T14:39:35.006832823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:39:44.886436967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:46968"
level=debug ts=2026-04-19T14:39:59.142246238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:40:01.287672986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38484"
ts=2026-04-19T14:40:05.010377536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:40:31.288918636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43454"
ts=2026-04-19T14:40:35.014111596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=debug ts=2026-04-19T14:40:35.746336669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:41:05.0176657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:41:08.77103848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57648"
level=debug ts=2026-04-19T14:41:20.355839938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:41:35.020137939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:41:56.612065451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:41:56.614138112Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:42:05.022272333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=info ts=2026-04-19T14:42:07.925309035Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-19T14:42:07.933737139Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:42:07.935591432Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:42:07.937737138Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:42:07.937767052Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-19T14:42:07.937778601Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-19T14:42:14.89764049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56668"
ts=2026-04-19T14:42:31.300208139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:54426"
ts=2026-04-19T14:42:35.024223278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:42:55.073288133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:43:05.027341204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:43:35.029277611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:43:38.784543305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55612"
level=debug ts=2026-04-19T14:43:42.145487834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:44:05.030120458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:44:14.909813443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:56826"
level=debug ts=2026-04-19T14:44:17.639949171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:44:35.032886391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:45:05.036338683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=debug ts=2026-04-19T14:45:12.887727455Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:45:31.317338044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:58964"
ts=2026-04-19T14:45:35.038519323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:46:05.039995162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:46:08.633161645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:46:08.635104024Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:46:35.042241214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T14:46:49.520049606Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:47:05.04570602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:47:08.80256643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43916"
ts=2026-04-19T14:47:35.049233454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T14:47:38.112478182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:48:05.052383146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:48:14.929321987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:58724"
level=debug ts=2026-04-19T14:48:17.820282764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:48:35.056393202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:48:44.933096822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52916"
ts=2026-04-19T14:49:05.059939965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T14:49:10.695672875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:49:35.063193111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:50:05.065513333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:50:08.819082915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50328"
level=debug ts=2026-04-19T14:50:12.092292013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:50:12.094053157Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:50:35.066441375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:51:01.353357164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:56086"
ts=2026-04-19T14:51:05.068866835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T14:51:09.776940998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:51:35.071405166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:51:38.829377584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47618"
level=debug ts=2026-04-19T14:51:49.734675048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:52:05.072631251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=error ts=2026-04-19T14:52:07.941676196Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:52:07.943913667Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:52:07.946056888Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T14:52:07.946081451Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:52:14.952203423Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:37532"
level=debug ts=2026-04-19T14:52:25.548077396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:52:31.361195752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:59560"
ts=2026-04-19T14:52:35.074591626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:53:01.362407324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44320"
ts=2026-04-19T14:53:05.07649192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:53:16.924887483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:53:31.364187039Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:45870"
ts=2026-04-19T14:53:35.078208609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:53:44.960280693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:40324"
level=debug ts=2026-04-19T14:53:52.688824744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:53:52.692197755Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:54:05.083177815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:54:08.843294211Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38652"
ts=2026-04-19T14:54:35.092419419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:54:45.714551833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:55:05.095525254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:55:24.200405774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:55:35.097471052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:55:44.970485568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:45086"
ts=2026-04-19T14:56:05.098919425Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:56:08.852880679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46464"
level=debug ts=2026-04-19T14:56:15.295649042Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:56:31.395589599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:49090"
ts=2026-04-19T14:56:35.101472973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T14:56:38.856653298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38794"
ts=2026-04-19T14:57:05.104261504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:57:08.85955298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34502"
level=debug ts=2026-04-19T14:57:14.659859905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:57:31.399624571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:60294"
ts=2026-04-19T14:57:35.107268158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:57:38.863201684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43012"
ts=2026-04-19T14:57:44.981440206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:46078"
level=debug ts=2026-04-19T14:57:51.49298216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T14:57:51.495547184Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T14:58:05.110736474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T14:58:34.821769261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:58:35.114288438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T14:59:05.11665071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
level=debug ts=2026-04-19T14:59:13.694776595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T14:59:14.998442944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:37672"
ts=2026-04-19T14:59:35.118270345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-77ec535f 10.129.0.56:7946"
ts=2026-04-19T14:59:38.877476259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:53198"
ts=2026-04-19T14:59:45.001334499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53840"
level=debug ts=2026-04-19T14:59:56.557837078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:00:05.120979723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:00:15.004614283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:57580"
ts=2026-04-19T15:00:35.124104005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T15:00:47.199153863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:00:48.343472964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:50906"
ts=2026-04-19T15:00:48.352960855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:50922"
ts=2026-04-19T15:01:03.907461837Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-77ec535f (timeout reached)"
ts=2026-04-19T15:01:05.127144682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:01:06.907711587Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-77ec535f has failed, no acks received"
level=debug ts=2026-04-19T15:01:23.566499663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T15:01:23.568346325Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:01:23.907939647Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-77ec535f (timeout reached)"
ts=2026-04-19T15:01:26.907458701Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-77ec535f has failed, no acks received"
ts=2026-04-19T15:01:35.130800001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:02:05.134510539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=debug ts=2026-04-19T15:02:07.926490179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T15:02:07.940713768Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:02:07.942637745Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:02:07.944430616Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:02:07.944455761Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:02:35.137768691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:02:38.897993067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:44432"
ts=2026-04-19T15:02:40.330826852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44138"
ts=2026-04-19T15:02:45.020060607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:53322"
level=debug ts=2026-04-19T15:03:03.398928781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:03:05.141266401Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:03:35.144711956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:03:38.908314245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59988"
level=debug ts=2026-04-19T15:03:51.189718624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:04:05.147693942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:04:10.338411285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:47670"
ts=2026-04-19T15:04:35.150091452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-abb5a05c 10.129.0.56:7946"
level=debug ts=2026-04-19T15:04:37.809560258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:04:45.034041476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:43460"
ts=2026-04-19T15:05:05.151406416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-abb5a05c 10.129.0.56:7946"
level=debug ts=2026-04-19T15:05:30.745068321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T15:05:30.746687272Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:05:35.153435668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:06:05.156418691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:06:15.042264027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:58302"
level=debug ts=2026-04-19T15:06:27.067214313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:06:35.158354836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-abb5a05c 10.129.0.56:7946"
ts=2026-04-19T15:07:05.160303017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:07:08.928202197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43570"
ts=2026-04-19T15:07:10.361644816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:39452"
level=debug ts=2026-04-19T15:07:20.92017522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:07:33.907930402Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-abb5a05c (timeout reached)"
ts=2026-04-19T15:07:35.163526379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:07:36.908290707Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-abb5a05c has failed, no acks received"
ts=2026-04-19T15:07:38.932111734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:36724"
ts=2026-04-19T15:07:48.907120211Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-abb5a05c (timeout reached)"
ts=2026-04-19T15:07:49.512957793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:37646"
ts=2026-04-19T15:07:49.522458916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:37660"
ts=2026-04-19T15:07:51.907355722Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-abb5a05c has failed, no acks received"
ts=2026-04-19T15:07:54.662972749Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-abb5a05c as failed, suspect timeout reached (2 peer confirmations)"
level=debug ts=2026-04-19T15:07:56.102235294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:08:05.166611001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:08:35.169291191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d483e31e 10.129.0.56:7946"
ts=2026-04-19T15:08:38.937781614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59314"
level=debug ts=2026-04-19T15:08:54.049215932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:09:05.171983661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:09:15.057614174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54792"
ts=2026-04-19T15:09:35.174563554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d483e31e 10.129.0.56:7946"
level=debug ts=2026-04-19T15:09:43.259859327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T15:09:43.261962813Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:10:05.177524049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=debug ts=2026-04-19T15:10:26.558963908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:10:35.183067589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d483e31e 10.129.0.56:7946"
ts=2026-04-19T15:10:45.067617878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:45502"
ts=2026-04-19T15:11:05.184580326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d483e31e 10.129.0.56:7946"
ts=2026-04-19T15:11:08.949870062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37238"
ts=2026-04-19T15:11:15.071189841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54760"
level=debug ts=2026-04-19T15:11:17.606703007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:11:35.186050318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d483e31e 10.129.0.56:7946"
ts=2026-04-19T15:11:38.952812142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50250"
level=debug ts=2026-04-19T15:11:51.74886923Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:12:05.188149781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=error ts=2026-04-19T15:12:07.942000313Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:12:07.944001814Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:12:07.945863384Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:12:07.945895164Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:12:17.117425854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:40436"
level=debug ts=2026-04-19T15:12:26.525717492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:12:35.191523838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:12:38.957916605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60816"
ts=2026-04-19T15:12:45.07927087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54866"
ts=2026-04-19T15:13:05.193490484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-d483e31e 10.129.0.56:7946"
level=debug ts=2026-04-19T15:13:10.93098486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T15:13:10.932714687Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:13:13.908186652Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d483e31e (timeout reached)"
ts=2026-04-19T15:13:16.907431593Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d483e31e has failed, no acks received"
ts=2026-04-19T15:13:28.908054027Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d483e31e (timeout reached)"
ts=2026-04-19T15:13:31.907951867Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d483e31e has failed, no acks received"
ts=2026-04-19T15:13:33.908900901Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-d483e31e (timeout reached)"
ts=2026-04-19T15:13:35.196256726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:13:36.908278066Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-d483e31e has failed, no acks received"
ts=2026-04-19T15:13:36.908393486Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-d483e31e as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T15:13:42.479098161Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38556"
ts=2026-04-19T15:13:42.487463928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38564"
ts=2026-04-19T15:14:05.19988416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=debug ts=2026-04-19T15:14:08.065733007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:14:15.087849934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54166"
ts=2026-04-19T15:14:35.203542406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:14:38.967103788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40262"
level=debug ts=2026-04-19T15:14:58.307226202Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:15:05.205865054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-836ea0b4 10.129.0.56:7946"
ts=2026-04-19T15:15:08.970172581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43168"
ts=2026-04-19T15:15:15.09499115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:36972"
ts=2026-04-19T15:15:35.208248848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:15:38.973871861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38864"
level=debug ts=2026-04-19T15:15:41.024634843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:16:05.211727661Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:16:08.978881341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48204"
level=debug ts=2026-04-19T15:16:29.479712054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:16:35.215287075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:16:38.982421708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43738"
ts=2026-04-19T15:17:05.218526249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T15:17:07.932308001Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T15:17:07.934180225Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:17:08.98652497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48166"
ts=2026-04-19T15:17:33.34154061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:40760"
ts=2026-04-19T15:17:35.221456458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=debug ts=2026-04-19T15:18:00.675772865Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:18:03.342690375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:56844"
ts=2026-04-19T15:18:05.225067096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:18:08.993081448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51650"
ts=2026-04-19T15:18:35.228497695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=debug ts=2026-04-19T15:18:46.506216284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:19:05.231904579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:19:33.35066229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:32898"
ts=2026-04-19T15:19:35.234932457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:19:38.99928991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47926"
level=debug ts=2026-04-19T15:19:41.961671184Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:20:03.351754769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:40240"
ts=2026-04-19T15:20:05.237327963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-836ea0b4 10.129.0.56:7946"
ts=2026-04-19T15:20:35.239383908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
level=debug ts=2026-04-19T15:20:42.377795538Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:21:05.241192173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-836ea0b4 10.129.0.56:7946"
ts=2026-04-19T15:21:15.124661402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:39676"
level=debug ts=2026-04-19T15:21:21.1009119Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T15:21:21.103246763Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:21:33.359249091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:55168"
ts=2026-04-19T15:21:35.2432902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:21:39.010534544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56368"
ts=2026-04-19T15:22:03.361036775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:49296"
ts=2026-04-19T15:22:05.246582066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
level=error ts=2026-04-19T15:22:07.941609608Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:22:07.943405895Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:22:07.945006004Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-19T15:22:07.945050919Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-19T15:22:13.908345036Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-836ea0b4 (timeout reached)"
ts=2026-04-19T15:22:15.131756943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:47356"
ts=2026-04-19T15:22:16.908280509Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-836ea0b4 has failed, no acks received"
level=debug ts=2026-04-19T15:22:20.517164384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:22:28.908194901Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-836ea0b4 (timeout reached)"
ts=2026-04-19T15:22:31.908621923Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-836ea0b4 has failed, no acks received"
ts=2026-04-19T15:22:35.249976067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:22:36.909047835Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-836ea0b4 as failed, suspect timeout reached (2 peer confirmations)"
ts=2026-04-19T15:22:38.908343958Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-836ea0b4 (timeout reached)"
ts=2026-04-19T15:22:39.017378843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59362"
ts=2026-04-19T15:22:41.908731693Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-836ea0b4 has failed, no acks received"
ts=2026-04-19T15:22:52.462063756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:48002"
ts=2026-04-19T15:22:52.470527752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:48010"
level=debug ts=2026-04-19T15:22:55.215171481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:23:05.25340292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:23:15.136358317Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:41690"
ts=2026-04-19T15:23:35.255541165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-a3806bcb 10.129.0.56:7946"
level=debug ts=2026-04-19T15:23:35.352806161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:24:05.258229773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:24:09.026489915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50670"
ts=2026-04-19T15:24:15.142577875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:40952"
level=debug ts=2026-04-19T15:24:22.337850607Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-19T15:24:35.260858591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-4c7tk-f47e0c42 10.128.0.67:7946"
ts=2026-04-19T15:24:45.146331471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:47898"
ts=2026-04-19T15:25:05.264454009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-7xb6j-5137a499 10.128.0.66:7946"
ts=2026-04-19T15:25:09.030277266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56584"
level=debug ts=2026-04-19T15:25:15.958120336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-19T15:25:15.95998136Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"