level=info ts=2026-04-16T19:07:18.241646088Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-16T19:07:18.2418518Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-04-16T19:07:18.242918728Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-16T19:07:18.243232197Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T19:07:18.243348581Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.80 inf=eth0
level=debug ts=2026-04-16T19:07:18.244314256Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-16T19:07:18.2443979Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.80 inf=eth0
level=info ts=2026-04-16T19:07:18.244422994Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-compactor-0-b347c337
ts=2026-04-16T19:07:18.244497688Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-16T19:07:18.244511377Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.80 advertisePort=7946
level=debug ts=2026-04-16T19:07:18.244528632Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.80 advertisePort=7946
level=info ts=2026-04-16T19:07:18.244687713Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-16T19:07:18.252376373Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T19:07:18.252396057Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-16T19:07:18.252408347Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=7.722616ms
level=error ts=2026-04-16T19:07:18.25241587Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-16T19:07:18.252428804Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-16T19:07:18.255717075Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T19:07:18.255739132Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-16T19:07:18.261988195Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=analytics
level=debug ts=2026-04-16T19:07:18.262009535Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-16T19:07:18.262019311Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-16T19:07:18.262026928Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-16T19:07:18.262110923Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=info ts=2026-04-16T19:07:18.262301087Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-16T19:07:18.262392204Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-16T19:07:18.2624028Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=runtime-config
level=debug ts=2026-04-16T19:07:18.262421459Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=info ts=2026-04-16T19:07:18.262425659Z caller=module_service.go:82 msg=starting module=compactor
level=info ts=2026-04-16T19:07:18.262457302Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=info ts=2026-04-16T19:07:18.262484922Z caller=basic_lifecycler.go:321 msg="instance not found in the ring" instance=loki-compactor-0 ring=compactor
level=info ts=2026-04-16T19:07:18.26251437Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-16T19:07:18.262631856Z caller=compactor.go:383 msg="waiting until compactor is JOINING in the ring"
level=info ts=2026-04-16T19:07:18.262648096Z caller=compactor.go:387 msg="compactor is JOINING in the ring"
level=debug ts=2026-04-16T19:07:18.262682064Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-16T19:07:19.263533782Z caller=compactor.go:397 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2026-04-16T19:07:19.38179655Z caller=compactor.go:401 msg="compactor is ACTIVE in the ring"
level=info ts=2026-04-16T19:07:19.381870624Z caller=loki.go:599 msg="Loki started" startup_time=1.217537852s
ts=2026-04-16T19:07:20.122665532Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T19:07:20.122696084Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:07:22.869725999Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T19:07:22.869752633Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-16T19:07:24.382654024Z caller=compactor.go:460 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2026-04-16T19:07:24.38270448Z caller=tables_manager.go:70 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
ts=2026-04-16T19:07:27.004322014Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-16T19:07:27.004344903Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-16T19:07:38.486291973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:07:40.30893819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:58248"
ts=2026-04-16T19:07:40.338618533Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.67:7946: dial tcp 10.130.0.67:7946: connect: connection refused"
ts=2026-04-16T19:07:40.339458106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.75:7946"
ts=2026-04-16T19:07:40.339944199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.80:7946"
ts=2026-04-16T19:07:40.3400137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.80:41758"
ts=2026-04-16T19:07:40.341248663Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.76:7946: dial tcp 10.129.0.76:7946: connect: connection refused"
ts=2026-04-16T19:07:40.341919624Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.78:7946: dial tcp 10.129.0.78:7946: connect: connection refused"
ts=2026-04-16T19:07:40.342483686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.79:7946"
level=info ts=2026-04-16T19:07:40.34295374Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=22.090519078s
ts=2026-04-16T19:07:40.504047856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45796"
level=debug ts=2026-04-16T19:07:52.59779103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:08:00.84583731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48668"
ts=2026-04-16T19:08:05.026086195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
level=debug ts=2026-04-16T19:08:11.379611316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:08:35.027340329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=debug ts=2026-04-16T19:08:36.02589774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:08:38.576826845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42658"
ts=2026-04-16T19:08:47.026975286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53084"
ts=2026-04-16T19:08:54.939440254Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:45346"
ts=2026-04-16T19:09:05.030047447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:09:05.438090481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T19:09:05.439845066Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:09:35.032847191Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:09:38.58281214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53982"
level=debug ts=2026-04-16T19:09:40.08639944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:10:05.035452377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:10:24.945516031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:46412"
ts=2026-04-16T19:10:35.03657596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
level=debug ts=2026-04-16T19:10:54.652192955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:11:05.038503778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:11:17.037942614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39836"
ts=2026-04-16T19:11:35.040292366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:11:47.038984468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56842"
ts=2026-04-16T19:11:54.950528667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:45478"
ts=2026-04-16T19:12:05.041514089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
level=debug ts=2026-04-16T19:12:09.726763092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:12:35.044140961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:13:05.04676422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:13:08.604187251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42408"
ts=2026-04-16T19:13:24.956620488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:37914"
level=debug ts=2026-04-16T19:13:27.281087283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-16T19:13:35.048711827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:13:38.607235196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35290"
ts=2026-04-16T19:13:47.047888045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49852"
ts=2026-04-16T19:14:05.050610326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:14:35.051736521Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=debug ts=2026-04-16T19:14:40.862543276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-16T19:14:40.864231917Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:14:47.051009535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35956"
ts=2026-04-16T19:15:05.053544291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:15:08.616406896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48694"
level=debug ts=2026-04-16T19:15:16.867372743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:15:24.963870948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:57412"
ts=2026-04-16T19:15:35.056108292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:15:38.620050467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56054"
ts=2026-04-16T19:15:47.053709631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:49972"
ts=2026-04-16T19:16:05.057655482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:16:08.623053935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55542"
level=debug ts=2026-04-16T19:16:17.284171533Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:16:35.058679238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:16:38.626054619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55986"
ts=2026-04-16T19:16:47.059088673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59082"
level=debug ts=2026-04-16T19:17:02.594420048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:17:05.0596911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:17:08.628936982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36920"
level=info ts=2026-04-16T19:17:24.383280919Z caller=tables_manager.go:76 msg="compactor startup delay completed"
level=error ts=2026-04-16T19:17:24.386440957Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:17:24.388429046Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:17:24.390411875Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:17:24.390449746Z caller=tables_manager.go:87 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-16T19:17:24.390462875Z caller=tables_manager.go:145 msg="compactor started"
ts=2026-04-16T19:17:35.062361951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:17:54.970880368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:40124"
level=debug ts=2026-04-16T19:17:57.760564384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:18:05.064440785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:18:08.635553482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42920"
ts=2026-04-16T19:18:17.063727707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35404"
ts=2026-04-16T19:18:35.065672715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:18:38.638360688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54072"
level=debug ts=2026-04-16T19:18:42.817434387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:18:42.818997383Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:19:05.067207696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=debug ts=2026-04-16T19:19:24.414474827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:19:35.068044622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:20:05.069431535Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:20:08.64839795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46214"
level=debug ts=2026-04-16T19:20:17.095485029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:20:35.072036987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:21:05.074094975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:21:08.654722562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39460"
level=debug ts=2026-04-16T19:21:12.063304615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:21:17.080001438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50400"
ts=2026-04-16T19:21:24.986985656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36468"
ts=2026-04-16T19:21:35.075643916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
level=debug ts=2026-04-16T19:21:57.421039388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:22:05.076788655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:22:35.078722413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:22:48.139809114Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:22:48.14163491Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:23:05.080592094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=debug ts=2026-04-16T19:23:29.194529869Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:23:35.081712299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:23:47.08907917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46922"
ts=2026-04-16T19:23:55.000370503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33500"
ts=2026-04-16T19:24:05.08274094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=debug ts=2026-04-16T19:24:23.64204866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:24:35.084132607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:24:38.676984699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40406"
ts=2026-04-16T19:24:47.091721177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:55160"
ts=2026-04-16T19:24:55.003678387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:56638"
level=debug ts=2026-04-16T19:25:03.961182861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:25:05.086504791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:25:08.679956018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60044"
ts=2026-04-16T19:25:17.092892827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37384"
ts=2026-04-16T19:25:25.005456969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33458"
ts=2026-04-16T19:25:35.08866344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:26:05.090515306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:26:06.01430959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:26:17.097077263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:56956"
ts=2026-04-16T19:26:35.092699387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
level=debug ts=2026-04-16T19:26:43.602878048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:26:43.60445Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:26:47.098873504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:40522"
ts=2026-04-16T19:27:05.094097931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=error ts=2026-04-16T19:27:24.395227458Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:27:24.397554741Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:27:24.39980194Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:27:24.399837927Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-16T19:27:33.67163892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:27:35.095961649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:27:38.696112679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53880"
ts=2026-04-16T19:27:55.017652311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:33626"
ts=2026-04-16T19:28:05.098185715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:28:10.566152685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:28:17.105546167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45990"
ts=2026-04-16T19:28:35.10177299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:29:04.860109503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:29:05.103951777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:29:08.705924207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48776"
ts=2026-04-16T19:29:35.10650039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:29:38.708861961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35590"
level=debug ts=2026-04-16T19:29:41.698110937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:30:05.111308604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:30:08.712753273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46810"
ts=2026-04-16T19:30:35.114033975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:30:39.892285248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:30:39.893723336Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:30:47.11496314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46850"
ts=2026-04-16T19:30:55.032513267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:60996"
ts=2026-04-16T19:31:05.117463908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:31:17.11678771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:36636"
ts=2026-04-16T19:31:35.120852584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:31:36.238301904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:32:05.123660929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:32:25.038530458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:56580"
level=debug ts=2026-04-16T19:32:28.044960467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:32:35.125396788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:32:47.121361567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46266"
ts=2026-04-16T19:33:05.126489446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=debug ts=2026-04-16T19:33:25.640902344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:33:35.127402242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:33:38.735982956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56838"
ts=2026-04-16T19:33:47.125946771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38110"
level=debug ts=2026-04-16T19:34:04.356417704Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:34:05.129509079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:34:35.133016121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:35:03.146158549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:35:03.148089723Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:35:05.13631742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:35:17.134613103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:45614"
ts=2026-04-16T19:35:35.138104798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=debug ts=2026-04-16T19:35:44.748570394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:36:05.140600473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:36:25.058070335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34534"
ts=2026-04-16T19:36:35.142498842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:36:38.75569531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39724"
level=debug ts=2026-04-16T19:36:44.9080817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:36:47.142784816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38044"
ts=2026-04-16T19:37:05.145357218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=error ts=2026-04-16T19:37:24.394681263Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:37:24.396611561Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:37:24.398594083Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:37:24.39862396Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:37:25.061366649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:34178"
ts=2026-04-16T19:37:35.148315494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:37:40.895904317Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:37:47.145426232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42820"
ts=2026-04-16T19:38:05.150140097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:38:35.152941373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:38:38.769024798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36602"
level=debug ts=2026-04-16T19:38:42.780508201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:38:55.066485941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:57212"
ts=2026-04-16T19:39:05.15543421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:39:08.771772026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51728"
level=debug ts=2026-04-16T19:39:26.514809434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:39:26.516713485Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:39:35.15794961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:39:38.774677639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51638"
ts=2026-04-16T19:40:05.160072768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=debug ts=2026-04-16T19:40:07.380120053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:40:35.162423265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:41:05.164392525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
level=debug ts=2026-04-16T19:41:08.731784283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:41:25.077394381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:49002"
ts=2026-04-16T19:41:35.166106215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:41:38.787339351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43402"
level=debug ts=2026-04-16T19:41:49.613858072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:42:05.167712723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:42:17.163452092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50206"
ts=2026-04-16T19:42:35.169416458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
level=debug ts=2026-04-16T19:42:40.269390802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:42:47.165445769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39522"
ts=2026-04-16T19:42:55.081156328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:36462"
ts=2026-04-16T19:43:05.171074165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:43:08.796225191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55008"
ts=2026-04-16T19:43:35.173237912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:43:41.877610809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:43:41.879494685Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:44:05.175721452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:44:17.171439197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:50634"
level=debug ts=2026-04-16T19:44:24.522007159Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:44:35.177601645Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:45:05.179203424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:45:17.176444158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:35386"
level=debug ts=2026-04-16T19:45:19.322464032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:45:35.180675439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:45:38.812974013Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46068"
level=debug ts=2026-04-16T19:45:57.390212081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:46:05.182017123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:46:35.183956801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:46:55.096756926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:47988"
level=debug ts=2026-04-16T19:46:58.77321404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:47:05.18636451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
level=error ts=2026-04-16T19:47:24.394719232Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:47:24.39662476Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:47:24.39838395Z caller=cached_client.go:190 msg="failed to build table names cache" err="operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=error ts=2026-04-16T19:47:24.398415602Z caller=tables_manager.go:101 msg="failed to run compaction" err="failed to list tables: operation error S3: ListObjectsV2, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data?delimiter=%2F&list-type=2&prefix=index%2F\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:47:25.098479454Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:41138"
ts=2026-04-16T19:47:35.188819424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:47:38.826897774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35434"
level=debug ts=2026-04-16T19:47:46.819621417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:47:46.821315082Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:47:55.099947838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:59864"
ts=2026-04-16T19:48:05.191433361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:48:17.186560458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:39902"
level=debug ts=2026-04-16T19:48:24.400117922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:48:35.19396603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:48:47.187529244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:59506"
level=debug ts=2026-04-16T19:49:04.157214466Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:49:05.195927982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:49:17.188661212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:42004"
ts=2026-04-16T19:49:35.196947076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:49:38.841098741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44434"
ts=2026-04-16T19:49:47.190471362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:53666"
ts=2026-04-16T19:49:55.106865541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:42432"
level=debug ts=2026-04-16T19:50:00.554975877Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:50:05.198052798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:50:08.8453515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59004"
ts=2026-04-16T19:50:35.200197326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
level=debug ts=2026-04-16T19:50:49.111991919Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:50:55.110373448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:44588"
ts=2026-04-16T19:51:05.203017239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:51:17.196445531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:38652"
ts=2026-04-16T19:51:35.204359427Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
level=debug ts=2026-04-16T19:51:39.668345122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-16T19:51:39.670078672Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-16T19:51:47.197640436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:46898"
ts=2026-04-16T19:52:05.205534439Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-j8j27-9afecd2e 10.129.0.75:7946"
ts=2026-04-16T19:52:08.859112245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59312"
ts=2026-04-16T19:52:25.122322865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:41084"
level=debug ts=2026-04-16T19:52:31.582887012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:52:35.20686112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:52:55.123653202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:38096"
ts=2026-04-16T19:53:05.207821372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:53:08.865505884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48174"
ts=2026-04-16T19:53:17.205308548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:37618"
level=debug ts=2026-04-16T19:53:25.03973904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:53:35.208835024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-mjlqb-e715ff6f 10.129.0.79:7946"
ts=2026-04-16T19:54:05.211661173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-ingester-0-9f8ced74 10.128.0.86:7946"
ts=2026-04-16T19:54:17.207567045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.75:32890"
level=debug ts=2026-04-16T19:54:18.474573556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-16T19:54:25.131293219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.79:58170"