level=info ts=2026-05-06T20:51:19.877222622Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-06T20:51:19.877257172Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-05-06T20:51:19.879752156Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-06T20:51:19.880097408Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T20:51:19.880167817Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.67 inf=eth0
level=debug ts=2026-05-06T20:51:19.880591636Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-06T20:51:19.880643767Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.67 inf=eth0
level=info ts=2026-05-06T20:51:19.880761625Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-f1a566ee
ts=2026-05-06T20:51:19.880853539Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-06T20:51:19.880868922Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.67 advertisePort=7946
level=debug ts=2026-05-06T20:51:19.880890391Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.67 advertisePort=7946
level=info ts=2026-05-06T20:51:19.881074514Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-05-06T20:51:19.887897766Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T20:51:19.887914513Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-06T20:51:19.887928078Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.856609ms
level=error ts=2026-05-06T20:51:19.887952251Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-06T20:51:19.887970451Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-06T20:51:19.896541843Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T20:51:19.896557179Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T20:51:19.899617292Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-06T20:51:19.899602877Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-06T20:51:19.899618728Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-05-06T20:51:19.899656015Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-06T20:51:19.899692106Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-05-06T20:51:19.899714624Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-06T20:51:19.899844439Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2026-05-06T20:51:19.899857763Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-05-06T20:51:19.900056252Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-05-06T20:51:19.900077718Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-05-06T20:51:19.900165385Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-06T20:51:19.900150477Z caller=basic_lifecycler.go:324 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-05-06T20:51:19.900175635Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-06T20:51:19.90030173Z caller=compactor.go:412 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-05-06T20:51:19.900329649Z caller=compactor.go:416 msg="compactor is JOINING in the ring"
level=debug ts=2026-05-06T20:51:19.900415578Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-06T20:51:20.901319035Z caller=compactor.go:426 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-05-06T20:51:21.002578554Z caller=compactor.go:430 msg="compactor is ACTIVE in the ring"
level=info ts=2026-05-06T20:51:21.002626701Z caller=loki.go:615 msg="Loki started" startup_time=1.189910483s
ts=2026-05-06T20:51:21.388502092Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T20:51:21.388522182Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T20:51:24.565386694Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-06T20:51:24.565410829Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-06T20:51:26.003170695Z caller=compactor.go:489 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-05-06T20:51:26.003225258Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-05-06T20:51:31.770919043Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.53:7946: dial tcp 10.130.0.53:7946: connect: connection refused"
ts=2026-05-06T20:51:31.771540854Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.51:7946: dial tcp 10.130.0.51:7946: connect: connection refused"
ts=2026-05-06T20:51:31.772057992Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.52:7946: dial tcp 10.130.0.52:7946: connect: connection refused"
ts=2026-05-06T20:51:31.773780779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.67:7946"
level=info ts=2026-05-06T20:51:31.775566838Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=11.887590378s
level=debug ts=2026-05-06T20:51:36.989066159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-06T20:51:54.410453004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T20:52:00.468672943Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54542"
ts=2026-05-06T20:52:05.225698801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:45488"
ts=2026-05-06T20:52:08.324247212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wnknt-4ea1d3bf 10.129.0.65:7946"
level=debug ts=2026-05-06T20:52:13.812655439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-06T20:52:36.67755471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T20:52:38.325220067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
level=debug ts=2026-05-06T20:52:59.122870363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T20:52:59.124433778Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T20:53:08.327187807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
ts=2026-05-06T20:53:35.230573919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:56688"
ts=2026-05-06T20:53:38.328972216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gvgfl-64a6a156 10.128.0.67:7946"
level=debug ts=2026-05-06T20:53:48.038302099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T20:54:00.481170829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54660"
ts=2026-05-06T20:54:03.669297253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:42336"
ts=2026-05-06T20:54:08.330760579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
ts=2026-05-06T20:54:38.333072355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gvgfl-64a6a156 10.128.0.67:7946"
level=debug ts=2026-05-06T20:54:48.954520451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T20:55:08.336066527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gvgfl-64a6a156 10.128.0.67:7946"
ts=2026-05-06T20:55:30.4906651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:47396"
ts=2026-05-06T20:55:33.674136886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:48646"
ts=2026-05-06T20:55:38.338188309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
ts=2026-05-06T20:56:05.242293765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:58428"
level=debug ts=2026-05-06T20:56:06.585552266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T20:56:08.3396805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
ts=2026-05-06T20:56:30.496901328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:44290"
ts=2026-05-06T20:56:38.340896956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wnknt-4ea1d3bf 10.129.0.65:7946"
ts=2026-05-06T20:57:00.499577936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:55030"
ts=2026-05-06T20:57:08.343495388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gvgfl-64a6a156 10.128.0.67:7946"
level=debug ts=2026-05-06T20:57:08.635530201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-06T20:57:38.346045736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
ts=2026-05-06T20:58:00.505762515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:47310"
ts=2026-05-06T20:58:03.682601457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:38350"
ts=2026-05-06T20:58:05.250187085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43830"
ts=2026-05-06T20:58:08.34725913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
level=debug ts=2026-05-06T20:58:14.004238839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-06T20:58:14.006183322Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T20:58:38.348730069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
level=debug ts=2026-05-06T20:59:00.360611125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T20:59:05.252658591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:46378"
ts=2026-05-06T20:59:08.350180812Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wnknt-4ea1d3bf 10.129.0.65:7946"
ts=2026-05-06T20:59:30.515195026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:55978"
ts=2026-05-06T20:59:38.35161272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
level=debug ts=2026-05-06T20:59:50.108572036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T21:00:05.257040931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:53840"
ts=2026-05-06T21:00:08.353587793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
ts=2026-05-06T21:00:30.521804417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:52664"
ts=2026-05-06T21:00:38.355827446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gvgfl-64a6a156 10.128.0.67:7946"
level=debug ts=2026-05-06T21:00:42.478401601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T21:01:00.525178114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:50438"
ts=2026-05-06T21:01:08.358280775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
level=info ts=2026-05-06T21:01:26.003493818Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-05-06T21:01:26.007472484Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T21:01:26.009589895Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T21:01:26.011599502Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-05-06T21:01:26.01162969Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-06T21:01:26.011643819Z caller=tables_manager.go:148 msg="compactor started"
ts=2026-05-06T21:01:30.528957067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:45924"
ts=2026-05-06T21:01:33.700408254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:52392"
ts=2026-05-06T21:01:35.262916206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:35838"
ts=2026-05-06T21:01:38.360112856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wnknt-4ea1d3bf 10.129.0.65:7946"
level=debug ts=2026-05-06T21:01:41.870448518Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T21:02:08.362129482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gvgfl-64a6a156 10.128.0.67:7946"
ts=2026-05-06T21:02:38.364920635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
level=debug ts=2026-05-06T21:02:40.194103965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-06T21:02:40.195643093Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-06T21:03:03.707265627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:50114"
ts=2026-05-06T21:03:08.36668907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wnknt-4ea1d3bf 10.129.0.65:7946"
level=debug ts=2026-05-06T21:03:25.881607037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T21:03:33.708786557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:44232"
ts=2026-05-06T21:03:38.36762107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
ts=2026-05-06T21:04:05.270864159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:53878"
ts=2026-05-06T21:04:08.369462418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wnknt-4ea1d3bf 10.129.0.65:7946"
level=debug ts=2026-05-06T21:04:11.755873133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T21:04:38.370699492Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-6ef008d7 10.129.0.66:7946"
ts=2026-05-06T21:05:00.552898584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.67:54394"
ts=2026-05-06T21:05:03.712280312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.65:52932"
ts=2026-05-06T21:05:08.37202669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wnknt-4ea1d3bf 10.129.0.65:7946"
level=debug ts=2026-05-06T21:05:10.958303206Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T21:05:35.277179877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:58532"
ts=2026-05-06T21:05:38.374670737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gvgfl-64a6a156 10.128.0.67:7946"
level=debug ts=2026-05-06T21:06:02.701714727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-06T21:06:05.278669413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43106"
ts=2026-05-06T21:06:08.376587575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-wnknt-4ea1d3bf 10.129.0.65:7946"